content/browser/renderer_host/media/video_capture_controller.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/renderer_host/media/video_capture_controller.h"
   6
   7 #include <map>
   8 #include <set>
   9
  10 #include "base/bind.h"
  11 #include "base/command_line.h"
  12 #include "base/metrics/histogram.h"
  13 #include "base/metrics/sparse_histogram.h"
  14 #include "base/stl_util.h"
  15 #include "content/browser/renderer_host/media/media_stream_manager.h"
  16 #include "content/browser/renderer_host/media/video_capture_buffer_pool.h"
  17 #include "content/browser/renderer_host/media/video_capture_device_client.h"
  18 #include "content/browser/renderer_host/media/video_capture_manager.h"
  19 #include "content/common/gpu/client/gl_helper.h"
  20 #include "content/public/browser/browser_thread.h"
  21 #include "content/public/common/content_switches.h"
  22 #include "gpu/command_buffer/common/mailbox_holder.h"
  23 #include "media/base/video_frame.h"
  24
  25 #if !defined(OS_ANDROID)
  26 #include "content/browser/compositor/image_transport_factory.h"
  27 #endif
  28
  29 using media::VideoCaptureFormat;
  30 using media::VideoFrame;
  31 using media::VideoFrameMetadata;
  32
  33 namespace content {
  34
  35 namespace {
  36
  37 static const int kInfiniteRatio = 99999;
  38
  39 #define UMA_HISTOGRAM_ASPECT_RATIO(name, width, height) \
  40     UMA_HISTOGRAM_SPARSE_SLOWLY( \
  41         name, \
  42         (height) ? ((width) * 100) / (height) : kInfiniteRatio);
  43
  44 class SyncPointClientImpl : public VideoFrame::SyncPointClient {
  45  public:
  46   explicit SyncPointClientImpl(GLHelper* gl_helper) : gl_helper_(gl_helper) {}
  47   ~SyncPointClientImpl() override {}
  48   uint32 InsertSyncPoint() override { return gl_helper_->InsertSyncPoint(); }
  49   void WaitSyncPoint(uint32 sync_point) override {
  50     gl_helper_->WaitSyncPoint(sync_point);
  51   }
  52
  53  private:
  54   GLHelper* gl_helper_;
  55 };
  56
  57 void ReturnVideoFrame(const scoped_refptr<VideoFrame>& video_frame,
  58                       uint32 sync_point) {
  59   DCHECK_CURRENTLY_ON(BrowserThread::UI);
  60 #if defined(OS_ANDROID)
  61   NOTREACHED();
  62 #else
  63   GLHelper* gl_helper = ImageTransportFactory::GetInstance()->GetGLHelper();
  64   // UpdateReleaseSyncPoint() creates a new sync_point using |gl_helper|, so
  65   // wait the given |sync_point| using |gl_helper|.
  66   if (gl_helper) {
  67     gl_helper->WaitSyncPoint(sync_point);
  68     SyncPointClientImpl client(gl_helper);
  69     video_frame->UpdateReleaseSyncPoint(&client);
  70   }
  71 #endif
  72 }
  73
  74 }  // anonymous namespace
  75
  76 struct VideoCaptureController::ControllerClient {
  77   ControllerClient(VideoCaptureControllerID id,
  78                    VideoCaptureControllerEventHandler* handler,
  79                    base::ProcessHandle render_process,
  80                    media::VideoCaptureSessionId session_id,
  81                    const media::VideoCaptureParams& params)
  82       : controller_id(id),
  83         event_handler(handler),
  84         render_process_handle(render_process),
  85         session_id(session_id),
  86         parameters(params),
  87         session_closed(false),
  88         paused(false) {}
  89
  90   ~ControllerClient() {}
  91
  92   // ID used for identifying this object.
  93   const VideoCaptureControllerID controller_id;
  94   VideoCaptureControllerEventHandler* const event_handler;
  95
  96   // Handle to the render process that will receive the capture buffers.
  97   const base::ProcessHandle render_process_handle;
  98   const media::VideoCaptureSessionId session_id;
  99   const media::VideoCaptureParams parameters;
 100
 101   // Buffers that are currently known to this client.
 102   std::set<int> known_buffers;
 103
 104   // Buffers currently held by this client, and syncpoint callback to call when
 105   // they are returned from the client.
 106   typedef std::map<int, scoped_refptr<VideoFrame>> ActiveBufferMap;
 107   ActiveBufferMap active_buffers;
 108
 109   // State of capture session, controlled by VideoCaptureManager directly. This
 110   // transitions to true as soon as StopSession() occurs, at which point the
 111   // client is sent an OnEnded() event. However, because the client retains a
 112   // VideoCaptureController* pointer, its ControllerClient entry lives on until
 113   // it unregisters itself via RemoveClient(), which may happen asynchronously.
 114   //
 115   // TODO(nick): If we changed the semantics of VideoCaptureHost so that
 116   // OnEnded() events were processed synchronously (with the RemoveClient() done
 117   // implicitly), we could avoid tracking this state here in the Controller, and
 118   // simplify the code in both places.
 119   bool session_closed;
 120
 121   // Indicates whether the client is paused, if true, VideoCaptureController
 122   // stops updating its buffer.
 123   bool paused;
 124 };
 125
 126 VideoCaptureController::VideoCaptureController(int max_buffers)
 127     : buffer_pool_(new VideoCaptureBufferPool(max_buffers)),
 128       state_(VIDEO_CAPTURE_STATE_STARTED),
 129       has_received_frames_(false),
 130       weak_ptr_factory_(this) {
 131   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 132 }
 133
 134 base::WeakPtr<VideoCaptureController>
 135 VideoCaptureController::GetWeakPtrForIOThread() {
 136   return weak_ptr_factory_.GetWeakPtr();
 137 }
 138
 139 scoped_ptr<media::VideoCaptureDevice::Client>
 140 VideoCaptureController::NewDeviceClient(
 141     const scoped_refptr<base::SingleThreadTaskRunner>& capture_task_runner) {
 142   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 143   return make_scoped_ptr(new VideoCaptureDeviceClient(
 144       this->GetWeakPtrForIOThread(), buffer_pool_, capture_task_runner));
 145 }
 146
 147 void VideoCaptureController::AddClient(
 148     VideoCaptureControllerID id,
 149     VideoCaptureControllerEventHandler* event_handler,
 150     base::ProcessHandle render_process,
 151     media::VideoCaptureSessionId session_id,
 152     const media::VideoCaptureParams& params) {
 153   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 154   DVLOG(1) << "VideoCaptureController::AddClient, id " << id
 155            << ", " << params.requested_format.frame_size.ToString()
 156            << ", " << params.requested_format.frame_rate
 157            << ", " << session_id
 158            << ")";
 159
 160   // If this is the first client added to the controller, cache the parameters.
 161   if (!controller_clients_.size())
 162     video_capture_format_ = params.requested_format;
 163
 164   // Signal error in case device is already in error state.
 165   if (state_ == VIDEO_CAPTURE_STATE_ERROR) {
 166     event_handler->OnError(id);
 167     return;
 168   }
 169
 170   // Do nothing if this client has called AddClient before.
 171   if (FindClient(id, event_handler, controller_clients_))
 172     return;
 173
 174   ControllerClient* client = new ControllerClient(
 175       id, event_handler, render_process, session_id, params);
 176   // If we already have gotten frame_info from the device, repeat it to the new
 177   // client.
 178   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
 179     controller_clients_.push_back(client);
 180     return;
 181   }
 182 }
 183
 184 int VideoCaptureController::RemoveClient(
 185     VideoCaptureControllerID id,
 186     VideoCaptureControllerEventHandler* event_handler) {
 187   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 188   DVLOG(1) << "VideoCaptureController::RemoveClient, id " << id;
 189
 190   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 191   if (!client)
 192     return kInvalidMediaCaptureSessionId;
 193
 194   // Take back all buffers held by the |client|.
 195   for (const auto& buffer : client->active_buffers)
 196     buffer_pool_->RelinquishConsumerHold(buffer.first, 1);
 197   client->active_buffers.clear();
 198
 199   int session_id = client->session_id;
 200   controller_clients_.remove(client);
 201   delete client;
 202
 203   return session_id;
 204 }
 205
 206 void VideoCaptureController::PauseClient(
 207     VideoCaptureControllerID id,
 208     VideoCaptureControllerEventHandler* event_handler) {
 209   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 210   DVLOG(1) << "VideoCaptureController::PauseClient, id " << id;
 211
 212   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 213   if (!client)
 214     return;
 215
 216   DLOG_IF(WARNING, client->paused) << "Redundant client configuration";
 217
 218   client->paused = true;
 219 }
 220
 221 bool VideoCaptureController::ResumeClient(
 222     VideoCaptureControllerID id,
 223     VideoCaptureControllerEventHandler* event_handler) {
 224   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 225   DVLOG(1) << "VideoCaptureController::ResumeClient, id " << id;
 226
 227   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 228   if (!client)
 229     return false;
 230
 231   if (!client->paused) {
 232     DVLOG(1) << "Calling resume on unpaused client";
 233     return false;
 234   }
 235
 236   client->paused = false;
 237   return true;
 238 }
 239
 240 void VideoCaptureController::StopSession(int session_id) {
 241   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 242   DVLOG(1) << "VideoCaptureController::StopSession, id " << session_id;
 243
 244   ControllerClient* client = FindClient(session_id, controller_clients_);
 245
 246   if (client) {
 247     client->session_closed = true;
 248     client->event_handler->OnEnded(client->controller_id);
 249   }
 250 }
 251
 252 void VideoCaptureController::ReturnBuffer(
 253     VideoCaptureControllerID id,
 254     VideoCaptureControllerEventHandler* event_handler,
 255     int buffer_id,
 256     uint32 sync_point,
 257     double consumer_resource_utilization) {
 258   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 259
 260   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 261
 262   // If this buffer is not held by this client, or this client doesn't exist
 263   // in controller, do nothing.
 264   ControllerClient::ActiveBufferMap::iterator iter;
 265   if (!client || (iter = client->active_buffers.find(buffer_id)) ==
 266                      client->active_buffers.end()) {
 267     NOTREACHED();
 268     return;
 269   }
 270
 271   // Set the RESOURCE_UTILIZATION to the maximum of those provided by each
 272   // consumer (via separate calls to this method that refer to the same
 273   // VideoFrame).  The producer of this VideoFrame may check this value, after
 274   // all consumer holds are relinquished, to make quality versus performance
 275   // trade-off decisions.
 276   scoped_refptr<VideoFrame> frame = iter->second;
 277   if (std::isfinite(consumer_resource_utilization) &&
 278       consumer_resource_utilization >= 0.0) {
 279     double resource_utilization = -1.0;
 280     if (frame->metadata()->GetDouble(VideoFrameMetadata::RESOURCE_UTILIZATION,
 281                                      &resource_utilization)) {
 282       frame->metadata()->SetDouble(VideoFrameMetadata::RESOURCE_UTILIZATION,
 283                                    std::max(consumer_resource_utilization,
 284                                             resource_utilization));
 285     } else {
 286       frame->metadata()->SetDouble(VideoFrameMetadata::RESOURCE_UTILIZATION,
 287                                    consumer_resource_utilization);
 288     }
 289   }
 290
 291   client->active_buffers.erase(iter);
 292   buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
 293
 294 #if defined(OS_ANDROID)
 295   DCHECK_EQ(0u, sync_point);
 296 #endif
 297   if (sync_point)
 298     BrowserThread::PostTask(BrowserThread::UI,
 299                             FROM_HERE,
 300                             base::Bind(&ReturnVideoFrame, frame, sync_point));
 301 }
 302
 303 const media::VideoCaptureFormat&
 304 VideoCaptureController::GetVideoCaptureFormat() const {
 305   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 306   return video_capture_format_;
 307 }
 308
 309 VideoCaptureController::~VideoCaptureController() {
 310   STLDeleteContainerPointers(controller_clients_.begin(),
 311                              controller_clients_.end());
 312 }
 313
 314 void VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread(
 315     scoped_ptr<media::VideoCaptureDevice::Client::Buffer> buffer,
 316     const scoped_refptr<VideoFrame>& frame,
 317     const base::TimeTicks& timestamp) {
 318   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 319   const int buffer_id = buffer->id();
 320   DCHECK_NE(buffer_id, VideoCaptureBufferPool::kInvalidId);
 321
 322   int count = 0;
 323   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
 324     if (!frame->metadata()->HasKey(VideoFrameMetadata::FRAME_RATE)) {
 325       frame->metadata()->SetDouble(VideoFrameMetadata::FRAME_RATE,
 326                                    video_capture_format_.frame_rate);
 327     }
 328     scoped_ptr<base::DictionaryValue> metadata(new base::DictionaryValue());
 329     frame->metadata()->MergeInternalValuesInto(metadata.get());
 330
 331     DCHECK(
 332         (frame->IsMappable() && frame->format() == media::PIXEL_FORMAT_I420) ||
 333         (frame->HasTextures() && (frame->format() == media::PIXEL_FORMAT_ARGB ||
 334                                   frame->format() == media::PIXEL_FORMAT_I420)))
 335         << "Format and/or storage type combination not supported (received: "
 336         << media::VideoPixelFormatToString(frame->format()) << ")";
 337
 338     for (const auto& client : controller_clients_) {
 339       if (client->session_closed || client->paused)
 340         continue;
 341
 342       // On the first use of a buffer on a client, share the memory handles.
 343       const bool is_new_buffer = client->known_buffers.insert(buffer_id).second;
 344       if (is_new_buffer) {
 345         if (frame->HasTextures()) {
 346           DCHECK(frame->coded_size() == frame->visible_rect().size())
 347               << "Textures are always supposed to be tightly packed.";
 348
 349           if (frame->format() == media::PIXEL_FORMAT_I420) {
 350             std::vector<gfx::GpuMemoryBufferHandle> handles(
 351                 VideoFrame::NumPlanes(frame->format()));
 352             for (size_t i = 0; i < handles.size(); ++i)
 353               buffer_pool_->ShareToProcess2(
 354                   buffer_id, i, client->render_process_handle, &handles[i]);
 355
 356             client->event_handler->OnBufferCreated2(
 357                 client->controller_id, handles, buffer->dimensions(),
 358                 buffer_id);
 359           } else {
 360             DCHECK_EQ(frame->format(), media::PIXEL_FORMAT_ARGB);
 361           }
 362         } else if (frame->IsMappable()) {
 363           DCHECK_EQ(frame->format(), media::PIXEL_FORMAT_I420);
 364           base::SharedMemoryHandle remote_handle;
 365           buffer_pool_->ShareToProcess(
 366               buffer_id, client->render_process_handle, &remote_handle);
 367
 368           client->event_handler->OnBufferCreated(
 369               client->controller_id, remote_handle, buffer->mapped_size(),
 370               buffer_id);
 371         }
 372       }
 373
 374       client->event_handler->OnBufferReady(client->controller_id,
 375                                            buffer_id,
 376                                            frame,
 377                                            timestamp);
 378       const bool inserted =
 379           client->active_buffers.insert(std::make_pair(buffer_id, frame))
 380               .second;
 381       DCHECK(inserted) << "Unexpected duplicate buffer: " << buffer_id;
 382       count++;
 383     }
 384   }
 385
 386   if (!has_received_frames_) {
 387     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Width",
 388                          frame->visible_rect().width());
 389     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Height",
 390                          frame->visible_rect().height());
 391     UMA_HISTOGRAM_ASPECT_RATIO("Media.VideoCapture.AspectRatio",
 392                                frame->visible_rect().width(),
 393                                frame->visible_rect().height());
 394     double frame_rate = 0.0f;
 395     if (!frame->metadata()->GetDouble(VideoFrameMetadata::FRAME_RATE,
 396                                       &frame_rate)) {
 397       frame_rate = video_capture_format_.frame_rate;
 398     }
 399     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.FrameRate", frame_rate);
 400     has_received_frames_ = true;
 401   }
 402
 403   buffer_pool_->HoldForConsumers(buffer_id, count);
 404 }
 405
 406 void VideoCaptureController::DoErrorOnIOThread() {
 407   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 408   state_ = VIDEO_CAPTURE_STATE_ERROR;
 409
 410   for (const auto* client : controller_clients_) {
 411     if (client->session_closed)
 412        continue;
 413     client->event_handler->OnError(client->controller_id);
 414   }
 415 }
 416
 417 void VideoCaptureController::DoLogOnIOThread(const std::string& message) {
 418   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 419   MediaStreamManager::SendMessageToNativeLog("Video capture: " + message);
 420 }
 421
 422 void VideoCaptureController::DoBufferDestroyedOnIOThread(
 423     int buffer_id_to_drop) {
 424   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 425
 426   for (auto* client : controller_clients_) {
 427     if (client->session_closed)
 428       continue;
 429
 430     if (client->known_buffers.erase(buffer_id_to_drop)) {
 431       client->event_handler->OnBufferDestroyed(client->controller_id,
 432                                                buffer_id_to_drop);
 433     }
 434   }
 435 }
 436
 437 VideoCaptureController::ControllerClient* VideoCaptureController::FindClient(
 438     VideoCaptureControllerID id,
 439     VideoCaptureControllerEventHandler* handler,
 440     const ControllerClients& clients) {
 441   for (auto* client : clients) {
 442     if (client->controller_id == id && client->event_handler == handler)
 443       return client;
 444   }
 445   return NULL;
 446 }
 447
 448 VideoCaptureController::ControllerClient* VideoCaptureController::FindClient(
 449     int session_id,
 450     const ControllerClients& clients) {
 451   for (auto client : clients) {
 452     if (client->session_id == session_id)
 453       return client;
 454   }
 455   return NULL;
 456 }
 457
 458 int VideoCaptureController::GetClientCount() const {
 459   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 460   return controller_clients_.size();
 461 }
 462
 463 int VideoCaptureController::GetActiveClientCount() const {
 464   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 465   int active_client_count = 0;
 466   for (ControllerClient* client : controller_clients_) {
 467     if (!client->paused)
 468       ++active_client_count;
 469   }
 470   return active_client_count;
 471 }
 472
 473 }  // namespace content