content/browser/renderer_host/media/video_capture_controller.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/renderer_host/media/video_capture_controller.h"
   6
   7 #include <map>
   8 #include <set>
   9
  10 #include "base/bind.h"
  11 #include "base/command_line.h"
  12 #include "base/metrics/histogram.h"
  13 #include "base/metrics/sparse_histogram.h"
  14 #include "base/stl_util.h"
  15 #include "content/browser/renderer_host/media/media_stream_manager.h"
  16 #include "content/browser/renderer_host/media/video_capture_buffer_pool.h"
  17 #include "content/browser/renderer_host/media/video_capture_device_client.h"
  18 #include "content/browser/renderer_host/media/video_capture_manager.h"
  19 #include "content/common/gpu/client/gl_helper.h"
  20 #include "content/public/browser/browser_thread.h"
  21 #include "content/public/common/content_switches.h"
  22 #include "gpu/command_buffer/common/mailbox_holder.h"
  23 #include "media/base/video_frame.h"
  24
  25 #if !defined(OS_ANDROID)
  26 #include "content/browser/compositor/image_transport_factory.h"
  27 #endif
  28
  29 using media::VideoCaptureFormat;
  30 using media::VideoFrame;
  31 using media::VideoFrameMetadata;
  32
  33 namespace content {
  34
  35 namespace {
  36
  37 static const int kInfiniteRatio = 99999;
  38
  39 #define UMA_HISTOGRAM_ASPECT_RATIO(name, width, height) \
  40     UMA_HISTOGRAM_SPARSE_SLOWLY( \
  41         name, \
  42         (height) ? ((width) * 100) / (height) : kInfiniteRatio);
  43
  44 class SyncPointClientImpl : public VideoFrame::SyncPointClient {
  45  public:
  46   explicit SyncPointClientImpl(GLHelper* gl_helper) : gl_helper_(gl_helper) {}
  47   ~SyncPointClientImpl() override {}
  48   uint32 InsertSyncPoint() override { return gl_helper_->InsertSyncPoint(); }
  49   void WaitSyncPoint(uint32 sync_point) override {
  50     gl_helper_->WaitSyncPoint(sync_point);
  51   }
  52
  53  private:
  54   GLHelper* gl_helper_;
  55 };
  56
  57 void ReturnVideoFrame(const scoped_refptr<VideoFrame>& video_frame,
  58                       uint32 sync_point) {
  59   DCHECK_CURRENTLY_ON(BrowserThread::UI);
  60 #if defined(OS_ANDROID)
  61   NOTREACHED();
  62 #else
  63   GLHelper* gl_helper = ImageTransportFactory::GetInstance()->GetGLHelper();
  64   // UpdateReleaseSyncPoint() creates a new sync_point using |gl_helper|, so
  65   // wait the given |sync_point| using |gl_helper|.
  66   if (gl_helper) {
  67     gl_helper->WaitSyncPoint(sync_point);
  68     SyncPointClientImpl client(gl_helper);
  69     video_frame->UpdateReleaseSyncPoint(&client);
  70   }
  71 #endif
  72 }
  73
  74 }  // anonymous namespace
  75
  76 struct VideoCaptureController::ControllerClient {
  77   ControllerClient(VideoCaptureControllerID id,
  78                    VideoCaptureControllerEventHandler* handler,
  79                    base::ProcessHandle render_process,
  80                    media::VideoCaptureSessionId session_id,
  81                    const media::VideoCaptureParams& params)
  82       : controller_id(id),
  83         event_handler(handler),
  84         render_process_handle(render_process),
  85         session_id(session_id),
  86         parameters(params),
  87         session_closed(false),
  88         paused(false) {}
  89
  90   ~ControllerClient() {}
  91
  92   // ID used for identifying this object.
  93   const VideoCaptureControllerID controller_id;
  94   VideoCaptureControllerEventHandler* const event_handler;
  95
  96   // Handle to the render process that will receive the capture buffers.
  97   const base::ProcessHandle render_process_handle;
  98   const media::VideoCaptureSessionId session_id;
  99   const media::VideoCaptureParams parameters;
 100
 101   // Buffers that are currently known to this client.
 102   std::set<int> known_buffers;
 103
 104   // Buffers currently held by this client, and syncpoint callback to call when
 105   // they are returned from the client.
 106   typedef std::map<int, scoped_refptr<VideoFrame>> ActiveBufferMap;
 107   ActiveBufferMap active_buffers;
 108
 109   // State of capture session, controlled by VideoCaptureManager directly. This
 110   // transitions to true as soon as StopSession() occurs, at which point the
 111   // client is sent an OnEnded() event. However, because the client retains a
 112   // VideoCaptureController* pointer, its ControllerClient entry lives on until
 113   // it unregisters itself via RemoveClient(), which may happen asynchronously.
 114   //
 115   // TODO(nick): If we changed the semantics of VideoCaptureHost so that
 116   // OnEnded() events were processed synchronously (with the RemoveClient() done
 117   // implicitly), we could avoid tracking this state here in the Controller, and
 118   // simplify the code in both places.
 119   bool session_closed;
 120
 121   // Indicates whether the client is paused, if true, VideoCaptureController
 122   // stops updating its buffer.
 123   bool paused;
 124 };
 125
 126 VideoCaptureController::VideoCaptureController(int max_buffers)
 127     : buffer_pool_(new VideoCaptureBufferPool(max_buffers)),
 128       state_(VIDEO_CAPTURE_STATE_STARTED),
 129       has_received_frames_(false),
 130       weak_ptr_factory_(this) {
 131   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 132 }
 133
 134 base::WeakPtr<VideoCaptureController>
 135 VideoCaptureController::GetWeakPtrForIOThread() {
 136   return weak_ptr_factory_.GetWeakPtr();
 137 }
 138
 139 scoped_ptr<media::VideoCaptureDevice::Client>
 140 VideoCaptureController::NewDeviceClient(
 141     const scoped_refptr<base::SingleThreadTaskRunner>& capture_task_runner) {
 142   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 143   return make_scoped_ptr(new VideoCaptureDeviceClient(
 144       this->GetWeakPtrForIOThread(), buffer_pool_, capture_task_runner));
 145 }
 146
 147 void VideoCaptureController::AddClient(
 148     VideoCaptureControllerID id,
 149     VideoCaptureControllerEventHandler* event_handler,
 150     base::ProcessHandle render_process,
 151     media::VideoCaptureSessionId session_id,
 152     const media::VideoCaptureParams& params) {
 153   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 154   DVLOG(1) << "VideoCaptureController::AddClient, id " << id
 155            << ", " << params.requested_format.frame_size.ToString()
 156            << ", " << params.requested_format.frame_rate
 157            << ", " << session_id
 158            << ")";
 159
 160   // If this is the first client added to the controller, cache the parameters.
 161   if (!controller_clients_.size())
 162     video_capture_format_ = params.requested_format;
 163
 164   // Signal error in case device is already in error state.
 165   if (state_ == VIDEO_CAPTURE_STATE_ERROR) {
 166     event_handler->OnError(id);
 167     return;
 168   }
 169
 170   // Do nothing if this client has called AddClient before.
 171   if (FindClient(id, event_handler, controller_clients_))
 172     return;
 173
 174   ControllerClient* client = new ControllerClient(
 175       id, event_handler, render_process, session_id, params);
 176   // If we already have gotten frame_info from the device, repeat it to the new
 177   // client.
 178   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
 179     controller_clients_.push_back(client);
 180     return;
 181   }
 182 }
 183
 184 int VideoCaptureController::RemoveClient(
 185     VideoCaptureControllerID id,
 186     VideoCaptureControllerEventHandler* event_handler) {
 187   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 188   DVLOG(1) << "VideoCaptureController::RemoveClient, id " << id;
 189
 190   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 191   if (!client)
 192     return kInvalidMediaCaptureSessionId;
 193
 194   // Take back all buffers held by the |client|.
 195   for (const auto& buffer : client->active_buffers)
 196     buffer_pool_->RelinquishConsumerHold(buffer.first, 1);
 197   client->active_buffers.clear();
 198
 199   int session_id = client->session_id;
 200   controller_clients_.remove(client);
 201   delete client;
 202
 203   return session_id;
 204 }
 205
 206 void VideoCaptureController::PauseOrResumeClient(
 207     VideoCaptureControllerID id,
 208     VideoCaptureControllerEventHandler* event_handler,
 209     bool pause) {
 210   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 211   DVLOG(1) << "VideoCaptureController::PauseOrResumeClient, id "
 212            << id << ", " << pause;
 213
 214   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 215   if (!client)
 216     return;
 217
 218   DLOG_IF(WARNING, client->paused == pause) << "Redundant client configuration";
 219   client->paused = pause;
 220 }
 221
 222 void VideoCaptureController::StopSession(int session_id) {
 223   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 224   DVLOG(1) << "VideoCaptureController::StopSession, id " << session_id;
 225
 226   ControllerClient* client = FindClient(session_id, controller_clients_);
 227
 228   if (client) {
 229     client->session_closed = true;
 230     client->event_handler->OnEnded(client->controller_id);
 231   }
 232 }
 233
 234 void VideoCaptureController::ReturnBuffer(
 235     VideoCaptureControllerID id,
 236     VideoCaptureControllerEventHandler* event_handler,
 237     int buffer_id,
 238     uint32 sync_point,
 239     double consumer_resource_utilization) {
 240   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 241
 242   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 243
 244   // If this buffer is not held by this client, or this client doesn't exist
 245   // in controller, do nothing.
 246   ControllerClient::ActiveBufferMap::iterator iter;
 247   if (!client || (iter = client->active_buffers.find(buffer_id)) ==
 248                      client->active_buffers.end()) {
 249     NOTREACHED();
 250     return;
 251   }
 252
 253   // Set the RESOURCE_UTILIZATION to the maximum of those provided by each
 254   // consumer (via separate calls to this method that refer to the same
 255   // VideoFrame).  The producer of this VideoFrame may check this value, after
 256   // all consumer holds are relinquished, to make quality versus performance
 257   // trade-off decisions.
 258   scoped_refptr<VideoFrame> frame = iter->second;
 259   if (std::isfinite(consumer_resource_utilization) &&
 260       consumer_resource_utilization >= 0.0) {
 261     double resource_utilization = -1.0;
 262     if (frame->metadata()->GetDouble(VideoFrameMetadata::RESOURCE_UTILIZATION,
 263                                      &resource_utilization)) {
 264       frame->metadata()->SetDouble(VideoFrameMetadata::RESOURCE_UTILIZATION,
 265                                    std::max(consumer_resource_utilization,
 266                                             resource_utilization));
 267     } else {
 268       frame->metadata()->SetDouble(VideoFrameMetadata::RESOURCE_UTILIZATION,
 269                                    consumer_resource_utilization);
 270     }
 271   }
 272
 273   client->active_buffers.erase(iter);
 274   buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
 275
 276 #if defined(OS_ANDROID)
 277   DCHECK_EQ(0u, sync_point);
 278 #endif
 279   if (sync_point)
 280     BrowserThread::PostTask(BrowserThread::UI,
 281                             FROM_HERE,
 282                             base::Bind(&ReturnVideoFrame, frame, sync_point));
 283 }
 284
 285 const media::VideoCaptureFormat&
 286 VideoCaptureController::GetVideoCaptureFormat() const {
 287   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 288   return video_capture_format_;
 289 }
 290
 291 VideoCaptureController::~VideoCaptureController() {
 292   STLDeleteContainerPointers(controller_clients_.begin(),
 293                              controller_clients_.end());
 294 }
 295
 296 void VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread(
 297     scoped_ptr<media::VideoCaptureDevice::Client::Buffer> buffer,
 298     const scoped_refptr<VideoFrame>& frame,
 299     const base::TimeTicks& timestamp) {
 300   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 301   const int buffer_id = buffer->id();
 302   DCHECK_NE(buffer_id, VideoCaptureBufferPool::kInvalidId);
 303
 304   int count = 0;
 305   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
 306     if (!frame->metadata()->HasKey(VideoFrameMetadata::FRAME_RATE)) {
 307       frame->metadata()->SetDouble(VideoFrameMetadata::FRAME_RATE,
 308                                    video_capture_format_.frame_rate);
 309     }
 310     scoped_ptr<base::DictionaryValue> metadata(new base::DictionaryValue());
 311     frame->metadata()->MergeInternalValuesInto(metadata.get());
 312
 313     for (const auto& client : controller_clients_) {
 314       if (client->session_closed || client->paused)
 315         continue;
 316
 317       CHECK((frame->IsMappable() &&
 318              frame->format() == media::PIXEL_FORMAT_I420) ||
 319             (!frame->IsMappable() && frame->HasTextures() &&
 320              frame->format() == media::PIXEL_FORMAT_ARGB))
 321           << "Format and/or storage type combination not supported (received: "
 322           << media::VideoPixelFormatToString(frame->format()) << ")";
 323
 324       if (frame->HasTextures()) {
 325         DCHECK(frame->coded_size() == frame->visible_rect().size())
 326             << "Textures are always supposed to be tightly packed.";
 327         DCHECK_EQ(1u, VideoFrame::NumPlanes(frame->format()));
 328       } else if (frame->format() == media::PIXEL_FORMAT_I420) {
 329         const bool is_new_buffer =
 330             client->known_buffers.insert(buffer_id).second;
 331         if (is_new_buffer) {
 332           // On the first use of a buffer on a client, share the memory handle.
 333           size_t memory_size = 0;
 334           base::SharedMemoryHandle remote_handle = buffer_pool_->ShareToProcess(
 335               buffer_id, client->render_process_handle, &memory_size);
 336           client->event_handler->OnBufferCreated(
 337               client->controller_id, remote_handle, memory_size, buffer_id);
 338         }
 339       }
 340
 341       client->event_handler->OnBufferReady(client->controller_id,
 342                                            buffer_id,
 343                                            frame,
 344                                            timestamp);
 345       const bool inserted =
 346           client->active_buffers.insert(std::make_pair(buffer_id, frame))
 347               .second;
 348       DCHECK(inserted) << "Unexpected duplicate buffer: " << buffer_id;
 349       count++;
 350     }
 351   }
 352
 353   if (!has_received_frames_) {
 354     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Width",
 355                          frame->visible_rect().width());
 356     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Height",
 357                          frame->visible_rect().height());
 358     UMA_HISTOGRAM_ASPECT_RATIO("Media.VideoCapture.AspectRatio",
 359                                frame->visible_rect().width(),
 360                                frame->visible_rect().height());
 361     double frame_rate = 0.0f;
 362     if (!frame->metadata()->GetDouble(VideoFrameMetadata::FRAME_RATE,
 363                                       &frame_rate)) {
 364       frame_rate = video_capture_format_.frame_rate;
 365     }
 366     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.FrameRate", frame_rate);
 367     has_received_frames_ = true;
 368   }
 369
 370   buffer_pool_->HoldForConsumers(buffer_id, count);
 371 }
 372
 373 void VideoCaptureController::DoErrorOnIOThread() {
 374   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 375   state_ = VIDEO_CAPTURE_STATE_ERROR;
 376
 377   for (const auto* client : controller_clients_) {
 378     if (client->session_closed)
 379        continue;
 380     client->event_handler->OnError(client->controller_id);
 381   }
 382 }
 383
 384 void VideoCaptureController::DoLogOnIOThread(const std::string& message) {
 385   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 386   MediaStreamManager::SendMessageToNativeLog("Video capture: " + message);
 387 }
 388
 389 void VideoCaptureController::DoBufferDestroyedOnIOThread(
 390     int buffer_id_to_drop) {
 391   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 392
 393   for (auto* client : controller_clients_) {
 394     if (client->session_closed)
 395       continue;
 396
 397     if (client->known_buffers.erase(buffer_id_to_drop)) {
 398       client->event_handler->OnBufferDestroyed(client->controller_id,
 399                                                buffer_id_to_drop);
 400     }
 401   }
 402 }
 403
 404 VideoCaptureController::ControllerClient* VideoCaptureController::FindClient(
 405     VideoCaptureControllerID id,
 406     VideoCaptureControllerEventHandler* handler,
 407     const ControllerClients& clients) {
 408   for (auto* client : clients) {
 409     if (client->controller_id == id && client->event_handler == handler)
 410       return client;
 411   }
 412   return NULL;
 413 }
 414
 415 VideoCaptureController::ControllerClient* VideoCaptureController::FindClient(
 416     int session_id,
 417     const ControllerClients& clients) {
 418   for (auto client : clients) {
 419     if (client->session_id == session_id)
 420       return client;
 421   }
 422   return NULL;
 423 }
 424
 425 int VideoCaptureController::GetClientCount() const {
 426   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 427   return controller_clients_.size();
 428 }
 429
 430 int VideoCaptureController::GetActiveClientCount() const {
 431   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 432   int active_client_count = 0;
 433   for (ControllerClient* client : controller_clients_) {
 434     if (!client->paused)
 435       ++active_client_count;
 436   }
 437   return active_client_count;
 438 }
 439
 440 }  // namespace content