content/browser/renderer_host/media/video_capture_controller.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/renderer_host/media/video_capture_controller.h"
   6
   7 #include <map>
   8 #include <set>
   9
  10 #include "base/bind.h"
  11 #include "base/command_line.h"
  12 #include "base/metrics/histogram.h"
  13 #include "base/metrics/sparse_histogram.h"
  14 #include "base/stl_util.h"
  15 #include "content/browser/renderer_host/media/media_stream_manager.h"
  16 #include "content/browser/renderer_host/media/video_capture_buffer_pool.h"
  17 #include "content/browser/renderer_host/media/video_capture_device_client.h"
  18 #include "content/browser/renderer_host/media/video_capture_manager.h"
  19 #include "content/common/gpu/client/gl_helper.h"
  20 #include "content/public/browser/browser_thread.h"
  21 #include "content/public/common/content_switches.h"
  22 #include "gpu/command_buffer/common/mailbox_holder.h"
  23 #include "media/base/video_frame.h"
  24
  25 #if !defined(OS_ANDROID)
  26 #include "content/browser/compositor/image_transport_factory.h"
  27 #endif
  28
  29 #if defined(ENABLE_WEBRTC) && (defined(OS_LINUX) || defined(OS_MACOSX))
  30 #include "content/browser/renderer_host/media/video_capture_texture_wrapper.h"
  31 #endif
  32
  33 using media::VideoCaptureFormat;
  34 using media::VideoFrame;
  35
  36 namespace content {
  37
  38 namespace {
  39
  40 static const int kInfiniteRatio = 99999;
  41
  42 #define UMA_HISTOGRAM_ASPECT_RATIO(name, width, height) \
  43     UMA_HISTOGRAM_SPARSE_SLOWLY( \
  44         name, \
  45         (height) ? ((width) * 100) / (height) : kInfiniteRatio);
  46
  47 class SyncPointClientImpl : public VideoFrame::SyncPointClient {
  48  public:
  49   explicit SyncPointClientImpl(GLHelper* gl_helper) : gl_helper_(gl_helper) {}
  50   ~SyncPointClientImpl() override {}
  51   uint32 InsertSyncPoint() override { return gl_helper_->InsertSyncPoint(); }
  52   void WaitSyncPoint(uint32 sync_point) override {
  53     gl_helper_->WaitSyncPoint(sync_point);
  54   }
  55
  56  private:
  57   GLHelper* gl_helper_;
  58 };
  59
  60 void ReturnVideoFrame(const scoped_refptr<VideoFrame>& video_frame,
  61                       uint32 sync_point) {
  62   DCHECK_CURRENTLY_ON(BrowserThread::UI);
  63 #if defined(OS_ANDROID)
  64   NOTREACHED();
  65 #else
  66   GLHelper* gl_helper = ImageTransportFactory::GetInstance()->GetGLHelper();
  67   // UpdateReleaseSyncPoint() creates a new sync_point using |gl_helper|, so
  68   // wait the given |sync_point| using |gl_helper|.
  69   if (gl_helper) {
  70     gl_helper->WaitSyncPoint(sync_point);
  71     SyncPointClientImpl client(gl_helper);
  72     video_frame->UpdateReleaseSyncPoint(&client);
  73   }
  74 #endif
  75 }
  76
  77 }  // anonymous namespace
  78
  79 struct VideoCaptureController::ControllerClient {
  80   ControllerClient(VideoCaptureControllerID id,
  81                    VideoCaptureControllerEventHandler* handler,
  82                    base::ProcessHandle render_process,
  83                    media::VideoCaptureSessionId session_id,
  84                    const media::VideoCaptureParams& params)
  85       : controller_id(id),
  86         event_handler(handler),
  87         render_process_handle(render_process),
  88         session_id(session_id),
  89         parameters(params),
  90         session_closed(false),
  91         paused(false) {}
  92
  93   ~ControllerClient() {}
  94
  95   // ID used for identifying this object.
  96   const VideoCaptureControllerID controller_id;
  97   VideoCaptureControllerEventHandler* const event_handler;
  98
  99   // Handle to the render process that will receive the capture buffers.
 100   const base::ProcessHandle render_process_handle;
 101   const media::VideoCaptureSessionId session_id;
 102   const media::VideoCaptureParams parameters;
 103
 104   // Buffers that are currently known to this client.
 105   std::set<int> known_buffers;
 106
 107   // Buffers currently held by this client, and syncpoint callback to call when
 108   // they are returned from the client.
 109   typedef std::map<int, scoped_refptr<VideoFrame>> ActiveBufferMap;
 110   ActiveBufferMap active_buffers;
 111
 112   // State of capture session, controlled by VideoCaptureManager directly. This
 113   // transitions to true as soon as StopSession() occurs, at which point the
 114   // client is sent an OnEnded() event. However, because the client retains a
 115   // VideoCaptureController* pointer, its ControllerClient entry lives on until
 116   // it unregisters itself via RemoveClient(), which may happen asynchronously.
 117   //
 118   // TODO(nick): If we changed the semantics of VideoCaptureHost so that
 119   // OnEnded() events were processed synchronously (with the RemoveClient() done
 120   // implicitly), we could avoid tracking this state here in the Controller, and
 121   // simplify the code in both places.
 122   bool session_closed;
 123
 124   // Indicates whether the client is paused, if true, VideoCaptureController
 125   // stops updating its buffer.
 126   bool paused;
 127 };
 128
 129 VideoCaptureController::VideoCaptureController(int max_buffers)
 130     : buffer_pool_(new VideoCaptureBufferPool(max_buffers)),
 131       state_(VIDEO_CAPTURE_STATE_STARTED),
 132       has_received_frames_(false),
 133       weak_ptr_factory_(this) {
 134   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 135 }
 136
 137 base::WeakPtr<VideoCaptureController>
 138 VideoCaptureController::GetWeakPtrForIOThread() {
 139   return weak_ptr_factory_.GetWeakPtr();
 140 }
 141
 142 scoped_ptr<media::VideoCaptureDevice::Client>
 143 VideoCaptureController::NewDeviceClient(
 144     const scoped_refptr<base::SingleThreadTaskRunner>& capture_task_runner,
 145     const media::VideoCaptureFormat& format) {
 146   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 147 #if defined(ENABLE_WEBRTC) && (defined(OS_LINUX) || defined(OS_MACOSX))
 148   if (base::CommandLine::ForCurrentProcess()->HasSwitch(
 149           switches::kEnableWebRtcCaptureToTexture)) {
 150     return make_scoped_ptr(new VideoCaptureTextureWrapper(
 151         this->GetWeakPtrForIOThread(), buffer_pool_, capture_task_runner,
 152         format));
 153     DVLOG(1) << "TextureWrapper, format " << format.ToString();
 154   }
 155 #endif
 156   return make_scoped_ptr(
 157       new VideoCaptureDeviceClient(this->GetWeakPtrForIOThread(),
 158                                    buffer_pool_));
 159 }
 160
 161 void VideoCaptureController::AddClient(
 162     VideoCaptureControllerID id,
 163     VideoCaptureControllerEventHandler* event_handler,
 164     base::ProcessHandle render_process,
 165     media::VideoCaptureSessionId session_id,
 166     const media::VideoCaptureParams& params) {
 167   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 168   DVLOG(1) << "VideoCaptureController::AddClient, id " << id
 169            << ", " << params.requested_format.frame_size.ToString()
 170            << ", " << params.requested_format.frame_rate
 171            << ", " << session_id
 172            << ")";
 173
 174   // If this is the first client added to the controller, cache the parameters.
 175   if (!controller_clients_.size())
 176     video_capture_format_ = params.requested_format;
 177
 178   // Signal error in case device is already in error state.
 179   if (state_ == VIDEO_CAPTURE_STATE_ERROR) {
 180     event_handler->OnError(id);
 181     return;
 182   }
 183
 184   // Do nothing if this client has called AddClient before.
 185   if (FindClient(id, event_handler, controller_clients_))
 186     return;
 187
 188   ControllerClient* client = new ControllerClient(
 189       id, event_handler, render_process, session_id, params);
 190   // If we already have gotten frame_info from the device, repeat it to the new
 191   // client.
 192   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
 193     controller_clients_.push_back(client);
 194     return;
 195   }
 196 }
 197
 198 int VideoCaptureController::RemoveClient(
 199     VideoCaptureControllerID id,
 200     VideoCaptureControllerEventHandler* event_handler) {
 201   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 202   DVLOG(1) << "VideoCaptureController::RemoveClient, id " << id;
 203
 204   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 205   if (!client)
 206     return kInvalidMediaCaptureSessionId;
 207
 208   // Take back all buffers held by the |client|.
 209   for (const auto& buffer : client->active_buffers)
 210     buffer_pool_->RelinquishConsumerHold(buffer.first, 1);
 211   client->active_buffers.clear();
 212
 213   int session_id = client->session_id;
 214   controller_clients_.remove(client);
 215   delete client;
 216
 217   return session_id;
 218 }
 219
 220 void VideoCaptureController::PauseOrResumeClient(
 221     VideoCaptureControllerID id,
 222     VideoCaptureControllerEventHandler* event_handler,
 223     bool pause) {
 224   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 225   DVLOG(1) << "VideoCaptureController::PauseOrResumeClient, id "
 226            << id << ", " << pause;
 227
 228   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 229   if (!client)
 230     return;
 231
 232   DCHECK(client->paused != pause);
 233   client->paused = pause;
 234 }
 235
 236 void VideoCaptureController::StopSession(int session_id) {
 237   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 238   DVLOG(1) << "VideoCaptureController::StopSession, id " << session_id;
 239
 240   ControllerClient* client = FindClient(session_id, controller_clients_);
 241
 242   if (client) {
 243     client->session_closed = true;
 244     client->event_handler->OnEnded(client->controller_id);
 245   }
 246 }
 247
 248 void VideoCaptureController::ReturnBuffer(
 249     VideoCaptureControllerID id,
 250     VideoCaptureControllerEventHandler* event_handler,
 251     int buffer_id,
 252     uint32 sync_point) {
 253   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 254
 255   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 256
 257   // If this buffer is not held by this client, or this client doesn't exist
 258   // in controller, do nothing.
 259   ControllerClient::ActiveBufferMap::iterator iter;
 260   if (!client || (iter = client->active_buffers.find(buffer_id)) ==
 261                      client->active_buffers.end()) {
 262     NOTREACHED();
 263     return;
 264   }
 265   scoped_refptr<VideoFrame> frame = iter->second;
 266   client->active_buffers.erase(iter);
 267   buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
 268
 269 #if defined(OS_ANDROID)
 270   DCHECK_EQ(0u, sync_point);
 271 #endif
 272   if (sync_point)
 273     BrowserThread::PostTask(BrowserThread::UI,
 274                             FROM_HERE,
 275                             base::Bind(&ReturnVideoFrame, frame, sync_point));
 276 }
 277
 278 const media::VideoCaptureFormat&
 279 VideoCaptureController::GetVideoCaptureFormat() const {
 280   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 281   return video_capture_format_;
 282 }
 283
 284 VideoCaptureController::~VideoCaptureController() {
 285   STLDeleteContainerPointers(controller_clients_.begin(),
 286                              controller_clients_.end());
 287 }
 288
 289 void VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread(
 290     const scoped_refptr<media::VideoCaptureDevice::Client::Buffer>& buffer,
 291     const scoped_refptr<VideoFrame>& frame,
 292     const base::TimeTicks& timestamp) {
 293   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 294   DCHECK_NE(buffer->id(), VideoCaptureBufferPool::kInvalidId);
 295
 296   int count = 0;
 297   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
 298     if (!frame->metadata()->HasKey(media::VideoFrameMetadata::FRAME_RATE)) {
 299       frame->metadata()->SetDouble(media::VideoFrameMetadata::FRAME_RATE,
 300                                    video_capture_format_.frame_rate);
 301     }
 302     scoped_ptr<base::DictionaryValue> metadata(new base::DictionaryValue());
 303     frame->metadata()->MergeInternalValuesInto(metadata.get());
 304
 305     for (const auto& client : controller_clients_) {
 306       if (client->session_closed || client->paused)
 307         continue;
 308
 309       scoped_ptr<base::DictionaryValue> copy_of_metadata;
 310       if (client == controller_clients_.back())
 311         copy_of_metadata = metadata.Pass();
 312       else
 313         copy_of_metadata.reset(metadata->DeepCopy());
 314
 315       if (frame->format() == VideoFrame::NATIVE_TEXTURE) {
 316         DCHECK(frame->coded_size() == frame->visible_rect().size())
 317             << "Textures are always supposed to be tightly packed.";
 318         client->event_handler->OnMailboxBufferReady(client->controller_id,
 319                                                     buffer->id(),
 320                                                     *frame->mailbox_holder(),
 321                                                     frame->coded_size(),
 322                                                     timestamp,
 323                                                     copy_of_metadata.Pass());
 324       } else if (frame->format() == media::VideoFrame::I420) {
 325         bool is_new_buffer = client->known_buffers.insert(buffer->id()).second;
 326         if (is_new_buffer) {
 327           // On the first use of a buffer on a client, share the memory handle.
 328           size_t memory_size = 0;
 329           base::SharedMemoryHandle remote_handle = buffer_pool_->ShareToProcess(
 330               buffer->id(), client->render_process_handle, &memory_size);
 331           client->event_handler->OnBufferCreated(
 332               client->controller_id, remote_handle, memory_size, buffer->id());
 333         }
 334
 335         client->event_handler->OnBufferReady(
 336             client->controller_id, buffer->id(), frame->coded_size(),
 337             frame->visible_rect(), timestamp, copy_of_metadata.Pass());
 338       } else {
 339         // VideoFrame format not supported.
 340         NOTREACHED();
 341         break;
 342       }
 343
 344       bool inserted =
 345           client->active_buffers.insert(std::make_pair(buffer->id(), frame))
 346               .second;
 347       DCHECK(inserted) << "Unexpected duplicate buffer: " << buffer->id();
 348       count++;
 349     }
 350   }
 351
 352   if (!has_received_frames_) {
 353     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Width",
 354                          frame->visible_rect().width());
 355     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Height",
 356                          frame->visible_rect().height());
 357     UMA_HISTOGRAM_ASPECT_RATIO("Media.VideoCapture.AspectRatio",
 358                                frame->visible_rect().width(),
 359                                frame->visible_rect().height());
 360     double frame_rate;
 361     if (!frame->metadata()->GetDouble(media::VideoFrameMetadata::FRAME_RATE,
 362                                       &frame_rate))
 363       frame_rate = video_capture_format_.frame_rate;
 364     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.FrameRate", frame_rate);
 365     has_received_frames_ = true;
 366   }
 367
 368   buffer_pool_->HoldForConsumers(buffer->id(), count);
 369 }
 370
 371 void VideoCaptureController::DoErrorOnIOThread() {
 372   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 373   state_ = VIDEO_CAPTURE_STATE_ERROR;
 374
 375   for (const auto* client : controller_clients_) {
 376     if (client->session_closed)
 377        continue;
 378     client->event_handler->OnError(client->controller_id);
 379   }
 380 }
 381
 382 void VideoCaptureController::DoLogOnIOThread(const std::string& message) {
 383   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 384   MediaStreamManager::SendMessageToNativeLog("Video capture: " + message);
 385 }
 386
 387 void VideoCaptureController::DoBufferDestroyedOnIOThread(
 388     int buffer_id_to_drop) {
 389   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 390
 391   for (auto* client : controller_clients_) {
 392     if (client->session_closed)
 393       continue;
 394
 395     if (client->known_buffers.erase(buffer_id_to_drop)) {
 396       client->event_handler->OnBufferDestroyed(client->controller_id,
 397                                                buffer_id_to_drop);
 398     }
 399   }
 400 }
 401
 402 VideoCaptureController::ControllerClient* VideoCaptureController::FindClient(
 403     VideoCaptureControllerID id,
 404     VideoCaptureControllerEventHandler* handler,
 405     const ControllerClients& clients) {
 406   for (auto* client : clients) {
 407     if (client->controller_id == id && client->event_handler == handler)
 408       return client;
 409   }
 410   return NULL;
 411 }
 412
 413 VideoCaptureController::ControllerClient* VideoCaptureController::FindClient(
 414     int session_id,
 415     const ControllerClients& clients) {
 416   for (auto client : clients) {
 417     if (client->session_id == session_id)
 418       return client;
 419   }
 420   return NULL;
 421 }
 422
 423 int VideoCaptureController::GetClientCount() const {
 424   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 425   return controller_clients_.size();
 426 }
 427
 428 int VideoCaptureController::GetActiveClientCount() const {
 429   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 430   int active_client_count = 0;
 431   for (ControllerClient* client : controller_clients_) {
 432     if (!client->paused)
 433       ++active_client_count;
 434   }
 435   return active_client_count;
 436 }
 437
 438 }  // namespace content