content/renderer/media/video_capture_impl.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // Notes about usage of this object by VideoCaptureImplManager.
   6 //
   7 // VideoCaptureImplManager access this object by using a Unretained()
   8 // binding and tasks on the IO thread. It is then important that
   9 // VideoCaptureImpl never post task to itself. All operations must be
  10 // synchronous.
  11
  12 #include "content/renderer/media/video_capture_impl.h"
  13
  14 #include "base/bind.h"
  15 #include "base/stl_util.h"
  16 #include "base/thread_task_runner_handle.h"
  17 #include "content/child/child_process.h"
  18 #include "content/common/media/video_capture_messages.h"
  19 #include "media/base/bind_to_current_loop.h"
  20 #include "media/base/limits.h"
  21 #include "media/base/video_frame.h"
  22
  23 namespace content {
  24
  25 namespace {
  26
  27 // This is called on an unknown thread when the VideoFrame destructor executes.
  28 // As of this writing, this callback mechanism is the only interface in
  29 // VideoFrame to provide the final value for |release_sync_point|.
  30 // VideoCaptureImpl::DidFinishConsumingFrame() will read the value saved here,
  31 // and pass it back to the IO thread to pass back to the host via the
  32 // BufferReady IPC.
  33 void SaveReleaseSyncPoint(uint32* storage, uint32 release_sync_point) {
  34   *storage = release_sync_point;
  35 }
  36
  37 }  // namespace
  38
  39 class VideoCaptureImpl::ClientBuffer
  40     : public base::RefCountedThreadSafe<ClientBuffer> {
  41  public:
  42   ClientBuffer(scoped_ptr<base::SharedMemory> buffer,
  43                size_t buffer_size)
  44       : buffer(buffer.Pass()),
  45         buffer_size(buffer_size) {}
  46   const scoped_ptr<base::SharedMemory> buffer;
  47   const size_t buffer_size;
  48
  49  private:
  50   friend class base::RefCountedThreadSafe<ClientBuffer>;
  51
  52   virtual ~ClientBuffer() {}
  53
  54   DISALLOW_COPY_AND_ASSIGN(ClientBuffer);
  55 };
  56
  57 VideoCaptureImpl::ClientInfo::ClientInfo() {}
  58 VideoCaptureImpl::ClientInfo::~ClientInfo() {}
  59
  60 VideoCaptureImpl::VideoCaptureImpl(
  61     const media::VideoCaptureSessionId session_id,
  62     VideoCaptureMessageFilter* filter)
  63     : message_filter_(filter),
  64       device_id_(0),
  65       session_id_(session_id),
  66       suspended_(false),
  67       state_(VIDEO_CAPTURE_STATE_STOPPED),
  68       weak_factory_(this) {
  69   DCHECK(filter);
  70 }
  71
  72 VideoCaptureImpl::~VideoCaptureImpl() {
  73   DCHECK(io_task_runner_->BelongsToCurrentThread());
  74 }
  75
  76 void VideoCaptureImpl::Init() {
  77   // For creating callbacks in unittest, this class may be constructed from a
  78   // different thread than the IO thread, e.g. wherever unittest runs on.
  79   // Therefore, this function should define the thread ownership.
  80 #if DCHECK_IS_ON()
  81   io_task_runner_ = base::ThreadTaskRunnerHandle::Get();
  82 #endif
  83   message_filter_->AddDelegate(this);
  84 }
  85
  86 void VideoCaptureImpl::DeInit() {
  87   DCHECK(io_task_runner_->BelongsToCurrentThread());
  88   if (state_ == VIDEO_CAPTURE_STATE_STARTED)
  89     Send(new VideoCaptureHostMsg_Stop(device_id_));
  90   message_filter_->RemoveDelegate(this);
  91 }
  92
  93 void VideoCaptureImpl::SuspendCapture(bool suspend) {
  94   DCHECK(io_task_runner_->BelongsToCurrentThread());
  95   Send(suspend ?
  96        static_cast<IPC::Message*>(new VideoCaptureHostMsg_Pause(device_id_)) :
  97        static_cast<IPC::Message*>(
  98            new VideoCaptureHostMsg_Resume(device_id_, session_id_, params_)));
  99 }
 100
 101 void VideoCaptureImpl::StartCapture(
 102     int client_id,
 103     const media::VideoCaptureParams& params,
 104     const VideoCaptureStateUpdateCB& state_update_cb,
 105     const VideoCaptureDeliverFrameCB& deliver_frame_cb) {
 106   DCHECK(io_task_runner_->BelongsToCurrentThread());
 107   ClientInfo client_info;
 108   client_info.params = params;
 109   client_info.state_update_cb = state_update_cb;
 110   client_info.deliver_frame_cb = deliver_frame_cb;
 111
 112   if (state_ == VIDEO_CAPTURE_STATE_ERROR) {
 113     state_update_cb.Run(VIDEO_CAPTURE_STATE_ERROR);
 114   } else if (clients_pending_on_filter_.count(client_id) ||
 115              clients_pending_on_restart_.count(client_id) ||
 116              clients_.count(client_id)) {
 117     LOG(FATAL) << "This client has already started.";
 118   } else if (!device_id_) {
 119     clients_pending_on_filter_[client_id] = client_info;
 120   } else {
 121     // Note: |state_| might not be started at this point. But we tell
 122     // client that we have started.
 123     state_update_cb.Run(VIDEO_CAPTURE_STATE_STARTED);
 124     if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
 125       clients_[client_id] = client_info;
 126       // TODO(sheu): Allowing resolution change will require that all
 127       // outstanding clients of a capture session support resolution change.
 128       DCHECK_EQ(params_.resolution_change_policy,
 129                 params.resolution_change_policy);
 130     } else if (state_ == VIDEO_CAPTURE_STATE_STOPPING) {
 131       clients_pending_on_restart_[client_id] = client_info;
 132       DVLOG(1) << "StartCapture: Got new resolution "
 133                << params.requested_format.frame_size.ToString()
 134                << " during stopping.";
 135     } else {
 136       clients_[client_id] = client_info;
 137       if (state_ == VIDEO_CAPTURE_STATE_STARTED)
 138         return;
 139       params_ = params;
 140       if (params_.requested_format.frame_rate >
 141           media::limits::kMaxFramesPerSecond) {
 142         params_.requested_format.frame_rate =
 143             media::limits::kMaxFramesPerSecond;
 144       }
 145       DVLOG(1) << "StartCapture: starting with first resolution "
 146                << params_.requested_format.frame_size.ToString();
 147       first_frame_timestamp_ = base::TimeTicks();
 148       StartCaptureInternal();
 149     }
 150   }
 151 }
 152
 153 void VideoCaptureImpl::StopCapture(int client_id) {
 154   DCHECK(io_task_runner_->BelongsToCurrentThread());
 155
 156   // A client ID can be in only one client list.
 157   // If this ID is in any client list, we can just remove it from
 158   // that client list and don't have to run the other following RemoveClient().
 159   if (!RemoveClient(client_id, &clients_pending_on_filter_)) {
 160     if (!RemoveClient(client_id, &clients_pending_on_restart_)) {
 161       RemoveClient(client_id, &clients_);
 162     }
 163   }
 164
 165   if (clients_.empty()) {
 166     DVLOG(1) << "StopCapture: No more client, stopping ...";
 167     StopDevice();
 168     client_buffers_.clear();
 169     weak_factory_.InvalidateWeakPtrs();
 170   }
 171 }
 172
 173 void VideoCaptureImpl::GetDeviceSupportedFormats(
 174     const VideoCaptureDeviceFormatsCB& callback) {
 175   DCHECK(io_task_runner_->BelongsToCurrentThread());
 176   device_formats_cb_queue_.push_back(callback);
 177   if (device_formats_cb_queue_.size() == 1)
 178     Send(new VideoCaptureHostMsg_GetDeviceSupportedFormats(device_id_,
 179                                                            session_id_));
 180 }
 181
 182 void VideoCaptureImpl::GetDeviceFormatsInUse(
 183     const VideoCaptureDeviceFormatsCB& callback) {
 184   DCHECK(io_task_runner_->BelongsToCurrentThread());
 185   device_formats_in_use_cb_queue_.push_back(callback);
 186   if (device_formats_in_use_cb_queue_.size() == 1)
 187     Send(
 188         new VideoCaptureHostMsg_GetDeviceFormatsInUse(device_id_, session_id_));
 189 }
 190
 191 void VideoCaptureImpl::OnBufferCreated(base::SharedMemoryHandle handle,
 192                                        int length,
 193                                        int buffer_id) {
 194   DCHECK(io_task_runner_->BelongsToCurrentThread());
 195
 196   // In case client calls StopCapture before the arrival of created buffer,
 197   // just close this buffer and return.
 198   if (state_ != VIDEO_CAPTURE_STATE_STARTED) {
 199     base::SharedMemory::CloseHandle(handle);
 200     return;
 201   }
 202
 203   scoped_ptr<base::SharedMemory> shm(new base::SharedMemory(handle, false));
 204   if (!shm->Map(length)) {
 205     DLOG(ERROR) << "OnBufferCreated: Map failed.";
 206     return;
 207   }
 208
 209   bool inserted =
 210       client_buffers_.insert(std::make_pair(
 211                                  buffer_id,
 212                                  new ClientBuffer(shm.Pass(),
 213                                                   length))).second;
 214   DCHECK(inserted);
 215 }
 216
 217 void VideoCaptureImpl::OnBufferDestroyed(int buffer_id) {
 218   DCHECK(io_task_runner_->BelongsToCurrentThread());
 219
 220   const ClientBufferMap::iterator iter = client_buffers_.find(buffer_id);
 221   if (iter == client_buffers_.end())
 222     return;
 223
 224   DCHECK(!iter->second.get() || iter->second->HasOneRef())
 225       << "Instructed to delete buffer we are still using.";
 226   client_buffers_.erase(iter);
 227 }
 228
 229 void VideoCaptureImpl::OnBufferReceived(
 230     int buffer_id,
 231     base::TimeTicks timestamp,
 232     const base::DictionaryValue& metadata,
 233     media::VideoPixelFormat pixel_format,
 234     media::VideoFrame::StorageType storage_type,
 235     const gfx::Size& coded_size,
 236     const gfx::Rect& visible_rect,
 237     const gpu::MailboxHolder& mailbox_holder) {
 238   DCHECK(io_task_runner_->BelongsToCurrentThread());
 239   if (state_ != VIDEO_CAPTURE_STATE_STARTED || suspended_) {
 240     Send(new VideoCaptureHostMsg_BufferReady(device_id_, buffer_id, 0, -1.0));
 241     return;
 242   }
 243   if (first_frame_timestamp_.is_null())
 244     first_frame_timestamp_ = timestamp;
 245
 246   // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
 247   TRACE_EVENT_INSTANT2("cast_perf_test", "OnBufferReceived",
 248                        TRACE_EVENT_SCOPE_THREAD, "timestamp",
 249                        timestamp.ToInternalValue(), "time_delta",
 250                        (timestamp - first_frame_timestamp_).ToInternalValue());
 251
 252   scoped_refptr<media::VideoFrame> frame;
 253   uint32* release_sync_point_storage = nullptr;
 254   scoped_refptr<ClientBuffer> buffer;
 255
 256   if (mailbox_holder.mailbox.IsZero()) {
 257     DCHECK_EQ(media::PIXEL_FORMAT_I420, pixel_format);
 258     const ClientBufferMap::const_iterator iter =
 259         client_buffers_.find(buffer_id);
 260     DCHECK(iter != client_buffers_.end());
 261     buffer = iter->second;
 262     frame = media::VideoFrame::WrapExternalSharedMemory(
 263         pixel_format,
 264         coded_size,
 265         visible_rect,
 266         gfx::Size(visible_rect.width(), visible_rect.height()),
 267         reinterpret_cast<uint8*>(buffer->buffer->memory()),
 268         buffer->buffer_size,
 269         buffer->buffer->handle(),
 270         0  /* shared_memory_offset */,
 271         timestamp - first_frame_timestamp_);
 272
 273   } else {
 274     DCHECK_EQ(media::PIXEL_FORMAT_ARGB, pixel_format);
 275     DCHECK(mailbox_holder.mailbox.Verify());  // Paranoia?
 276     // To be deleted in DidFinishConsumingFrame().
 277     release_sync_point_storage = new uint32(0);
 278     frame = media::VideoFrame::WrapNativeTexture(
 279         pixel_format,
 280         mailbox_holder,
 281         base::Bind(&SaveReleaseSyncPoint, release_sync_point_storage),
 282         coded_size,
 283         gfx::Rect(coded_size),
 284         coded_size,
 285         timestamp - first_frame_timestamp_);
 286   }
 287   frame->AddDestructionObserver(
 288       base::Bind(&VideoCaptureImpl::DidFinishConsumingFrame,
 289                  frame->metadata(),
 290                  release_sync_point_storage,
 291                  media::BindToCurrentLoop(base::Bind(
 292                      &VideoCaptureImpl::OnClientBufferFinished,
 293                      weak_factory_.GetWeakPtr(),
 294                      buffer_id,
 295                      buffer))));
 296
 297   frame->metadata()->MergeInternalValuesFrom(metadata);
 298
 299   for (const auto& client : clients_)
 300     client.second.deliver_frame_cb.Run(frame, timestamp);
 301 }
 302
 303 void VideoCaptureImpl::OnClientBufferFinished(
 304     int buffer_id,
 305     const scoped_refptr<ClientBuffer>& /* ignored_buffer */,
 306     uint32 release_sync_point,
 307     double consumer_resource_utilization) {
 308   DCHECK(io_task_runner_->BelongsToCurrentThread());
 309   Send(new VideoCaptureHostMsg_BufferReady(device_id_,
 310                                            buffer_id,
 311                                            release_sync_point,
 312                                            consumer_resource_utilization));
 313 }
 314
 315 void VideoCaptureImpl::OnStateChanged(VideoCaptureState state) {
 316   DCHECK(io_task_runner_->BelongsToCurrentThread());
 317
 318   switch (state) {
 319     case VIDEO_CAPTURE_STATE_STARTED:
 320       // Camera has started in the browser process. Since we have already
 321       // told all clients that we have started there's nothing to do.
 322       break;
 323     case VIDEO_CAPTURE_STATE_STOPPED:
 324       state_ = VIDEO_CAPTURE_STATE_STOPPED;
 325       DVLOG(1) << "OnStateChanged: stopped!, device_id = " << device_id_;
 326       client_buffers_.clear();
 327       weak_factory_.InvalidateWeakPtrs();
 328       if (!clients_.empty() || !clients_pending_on_restart_.empty())
 329         RestartCapture();
 330       break;
 331     case VIDEO_CAPTURE_STATE_PAUSED:
 332       for (const auto& client : clients_)
 333         client.second.state_update_cb.Run(VIDEO_CAPTURE_STATE_PAUSED);
 334       break;
 335     case VIDEO_CAPTURE_STATE_ERROR:
 336       DVLOG(1) << "OnStateChanged: error!, device_id = " << device_id_;
 337       for (const auto& client : clients_)
 338         client.second.state_update_cb.Run(VIDEO_CAPTURE_STATE_ERROR);
 339       clients_.clear();
 340       state_ = VIDEO_CAPTURE_STATE_ERROR;
 341       break;
 342     case VIDEO_CAPTURE_STATE_ENDED:
 343       DVLOG(1) << "OnStateChanged: ended!, device_id = " << device_id_;
 344       for (const auto& client : clients_) {
 345         // We'll only notify the client that the stream has stopped.
 346         client.second.state_update_cb.Run(VIDEO_CAPTURE_STATE_STOPPED);
 347       }
 348       clients_.clear();
 349       state_ = VIDEO_CAPTURE_STATE_ENDED;
 350       break;
 351     default:
 352       break;
 353   }
 354 }
 355
 356 void VideoCaptureImpl::OnDeviceSupportedFormatsEnumerated(
 357     const media::VideoCaptureFormats& supported_formats) {
 358   DCHECK(io_task_runner_->BelongsToCurrentThread());
 359   for (size_t i = 0; i < device_formats_cb_queue_.size(); ++i)
 360     device_formats_cb_queue_[i].Run(supported_formats);
 361   device_formats_cb_queue_.clear();
 362 }
 363
 364 void VideoCaptureImpl::OnDeviceFormatsInUseReceived(
 365     const media::VideoCaptureFormats& formats_in_use) {
 366   DCHECK(io_task_runner_->BelongsToCurrentThread());
 367   for (size_t i = 0; i < device_formats_in_use_cb_queue_.size(); ++i)
 368     device_formats_in_use_cb_queue_[i].Run(formats_in_use);
 369   device_formats_in_use_cb_queue_.clear();
 370 }
 371
 372 void VideoCaptureImpl::OnDelegateAdded(int32 device_id) {
 373   DCHECK(io_task_runner_->BelongsToCurrentThread());
 374   DVLOG(1) << "OnDelegateAdded: device_id " << device_id;
 375
 376   device_id_ = device_id;
 377   ClientInfoMap::iterator it = clients_pending_on_filter_.begin();
 378   while (it != clients_pending_on_filter_.end()) {
 379     const int client_id = it->first;
 380     const ClientInfo client_info = it->second;
 381     clients_pending_on_filter_.erase(it++);
 382     StartCapture(client_id, client_info.params, client_info.state_update_cb,
 383                  client_info.deliver_frame_cb);
 384   }
 385 }
 386
 387 void VideoCaptureImpl::StopDevice() {
 388   DCHECK(io_task_runner_->BelongsToCurrentThread());
 389
 390   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
 391     state_ = VIDEO_CAPTURE_STATE_STOPPING;
 392     Send(new VideoCaptureHostMsg_Stop(device_id_));
 393     params_.requested_format.frame_size.SetSize(0, 0);
 394   }
 395 }
 396
 397 void VideoCaptureImpl::RestartCapture() {
 398   DCHECK(io_task_runner_->BelongsToCurrentThread());
 399   DCHECK_EQ(state_, VIDEO_CAPTURE_STATE_STOPPED);
 400
 401   int width = 0;
 402   int height = 0;
 403   clients_.insert(clients_pending_on_restart_.begin(),
 404                   clients_pending_on_restart_.end());
 405   clients_pending_on_restart_.clear();
 406   for (const auto& client : clients_) {
 407     width = std::max(width,
 408                      client.second.params.requested_format.frame_size.width());
 409     height = std::max(
 410         height, client.second.params.requested_format.frame_size.height());
 411   }
 412   params_.requested_format.frame_size.SetSize(width, height);
 413   DVLOG(1) << "RestartCapture, "
 414            << params_.requested_format.frame_size.ToString();
 415   StartCaptureInternal();
 416 }
 417
 418 void VideoCaptureImpl::StartCaptureInternal() {
 419   DCHECK(io_task_runner_->BelongsToCurrentThread());
 420   DCHECK(device_id_);
 421
 422   Send(new VideoCaptureHostMsg_Start(device_id_, session_id_, params_));
 423   state_ = VIDEO_CAPTURE_STATE_STARTED;
 424 }
 425
 426 void VideoCaptureImpl::Send(IPC::Message* message) {
 427   DCHECK(io_task_runner_->BelongsToCurrentThread());
 428   message_filter_->Send(message);
 429 }
 430
 431 bool VideoCaptureImpl::RemoveClient(int client_id, ClientInfoMap* clients) {
 432   DCHECK(io_task_runner_->BelongsToCurrentThread());
 433   bool found = false;
 434
 435   const ClientInfoMap::iterator it = clients->find(client_id);
 436   if (it != clients->end()) {
 437     it->second.state_update_cb.Run(VIDEO_CAPTURE_STATE_STOPPED);
 438     clients->erase(it);
 439     found = true;
 440   }
 441   return found;
 442 }
 443
 444 // static
 445 void VideoCaptureImpl::DidFinishConsumingFrame(
 446     const media::VideoFrameMetadata* metadata,
 447     uint32* release_sync_point_storage,
 448     const base::Callback<void(uint32, double)>& callback_to_io_thread) {
 449   // Note: This function may be called on any thread by the VideoFrame
 450   // destructor.  |metadata| is still valid for read-access at this point.
 451
 452   uint32 release_sync_point = 0u;
 453   if (release_sync_point_storage) {
 454     release_sync_point = *release_sync_point_storage;
 455     delete release_sync_point_storage;
 456   }
 457
 458   double consumer_resource_utilization = -1.0;
 459   if (!metadata->GetDouble(media::VideoFrameMetadata::RESOURCE_UTILIZATION,
 460                            &consumer_resource_utilization)) {
 461     consumer_resource_utilization = -1.0;
 462   }
 463
 464   callback_to_io_thread.Run(release_sync_point, consumer_resource_utilization);
 465 }
 466
 467 }  // namespace content