1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/renderer_host/media/video_capture_controller.h"
10 #include "base/bind.h"
11 #include "base/debug/trace_event.h"
12 #include "base/metrics/histogram.h"
13 #include "base/metrics/sparse_histogram.h"
14 #include "base/stl_util.h"
15 #include "content/browser/renderer_host/media/media_stream_manager.h"
16 #include "content/browser/renderer_host/media/video_capture_manager.h"
17 #include "content/public/browser/browser_thread.h"
18 #include "gpu/command_buffer/common/mailbox_holder.h"
19 #include "media/base/video_frame.h"
20 #include "media/base/video_util.h"
21 #include "media/base/yuv_convert.h"
22 #include "third_party/libyuv/include/libyuv.h"
24 using media::VideoCaptureFormat
;
30 static const int kInfiniteRatio
= 99999;
32 #define UMA_HISTOGRAM_ASPECT_RATIO(name, width, height) \
33 UMA_HISTOGRAM_SPARSE_SLOWLY( \
35 (height) ? ((width) * 100) / (height) : kInfiniteRatio);
37 // The number of buffers that VideoCaptureBufferPool should allocate.
38 const int kNoOfBuffers
= 3;
40 class PoolBuffer
: public media::VideoCaptureDevice::Client::Buffer
{
42 PoolBuffer(const scoped_refptr
<VideoCaptureBufferPool
>& pool
,
46 : Buffer(buffer_id
, data
, size
), pool_(pool
) {
51 virtual ~PoolBuffer() { pool_
->RelinquishProducerReservation(id()); }
53 const scoped_refptr
<VideoCaptureBufferPool
> pool_
;
56 } // anonymous namespace
58 struct VideoCaptureController::ControllerClient
{
59 ControllerClient(const VideoCaptureControllerID
& id
,
60 VideoCaptureControllerEventHandler
* handler
,
61 base::ProcessHandle render_process
,
62 media::VideoCaptureSessionId session_id
,
63 const media::VideoCaptureParams
& params
)
65 event_handler(handler
),
66 render_process_handle(render_process
),
67 session_id(session_id
),
69 session_closed(false) {}
71 ~ControllerClient() {}
73 // ID used for identifying this object.
74 const VideoCaptureControllerID controller_id
;
75 VideoCaptureControllerEventHandler
* const event_handler
;
77 // Handle to the render process that will receive the capture buffers.
78 const base::ProcessHandle render_process_handle
;
79 const media::VideoCaptureSessionId session_id
;
80 const media::VideoCaptureParams parameters
;
82 // Buffers that are currently known to this client.
83 std::set
<int> known_buffers
;
85 // Buffers currently held by this client, and syncpoint callback to call when
86 // they are returned from the client.
87 typedef std::map
<int, scoped_refptr
<media::VideoFrame
> > ActiveBufferMap
;
88 ActiveBufferMap active_buffers
;
90 // State of capture session, controlled by VideoCaptureManager directly. This
91 // transitions to true as soon as StopSession() occurs, at which point the
92 // client is sent an OnEnded() event. However, because the client retains a
93 // VideoCaptureController* pointer, its ControllerClient entry lives on until
94 // it unregisters itself via RemoveClient(), which may happen asynchronously.
96 // TODO(nick): If we changed the semantics of VideoCaptureHost so that
97 // OnEnded() events were processed synchronously (with the RemoveClient() done
98 // implicitly), we could avoid tracking this state here in the Controller, and
99 // simplify the code in both places.
103 // Receives events from the VideoCaptureDevice and posts them to a
104 // VideoCaptureController on the IO thread. An instance of this class may safely
105 // outlive its target VideoCaptureController.
107 // Methods of this class may be called from any thread, and in practice will
108 // often be called on some auxiliary thread depending on the platform and the
109 // device type; including, for example, the DirectShow thread on Windows, the
110 // v4l2_thread on Linux, and the UI thread for tab capture.
111 class VideoCaptureController::VideoCaptureDeviceClient
112 : public media::VideoCaptureDevice::Client
{
114 explicit VideoCaptureDeviceClient(
115 const base::WeakPtr
<VideoCaptureController
>& controller
,
116 const scoped_refptr
<VideoCaptureBufferPool
>& buffer_pool
);
117 virtual ~VideoCaptureDeviceClient();
119 // VideoCaptureDevice::Client implementation.
120 virtual scoped_refptr
<Buffer
> ReserveOutputBuffer(
121 media::VideoFrame::Format format
,
122 const gfx::Size
& size
) OVERRIDE
;
123 virtual void OnIncomingCapturedData(const uint8
* data
,
125 const VideoCaptureFormat
& frame_format
,
127 base::TimeTicks timestamp
) OVERRIDE
;
128 virtual void OnIncomingCapturedVideoFrame(
129 const scoped_refptr
<Buffer
>& buffer
,
130 const VideoCaptureFormat
& buffer_format
,
131 const scoped_refptr
<media::VideoFrame
>& frame
,
132 base::TimeTicks timestamp
) OVERRIDE
;
133 virtual void OnError(const std::string
& reason
) OVERRIDE
;
136 scoped_refptr
<Buffer
> DoReserveOutputBuffer(media::VideoFrame::Format format
,
137 const gfx::Size
& dimensions
);
139 // The controller to which we post events.
140 const base::WeakPtr
<VideoCaptureController
> controller_
;
142 // The pool of shared-memory buffers used for capturing.
143 const scoped_refptr
<VideoCaptureBufferPool
> buffer_pool_
;
148 VideoCaptureController::VideoCaptureController()
149 : buffer_pool_(new VideoCaptureBufferPool(kNoOfBuffers
)),
150 state_(VIDEO_CAPTURE_STATE_STARTED
),
151 weak_ptr_factory_(this) {
154 VideoCaptureController::VideoCaptureDeviceClient::VideoCaptureDeviceClient(
155 const base::WeakPtr
<VideoCaptureController
>& controller
,
156 const scoped_refptr
<VideoCaptureBufferPool
>& buffer_pool
)
157 : controller_(controller
), buffer_pool_(buffer_pool
), first_frame_(true) {}
159 VideoCaptureController::VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {}
161 base::WeakPtr
<VideoCaptureController
> VideoCaptureController::GetWeakPtr() {
162 return weak_ptr_factory_
.GetWeakPtr();
165 scoped_ptr
<media::VideoCaptureDevice::Client
>
166 VideoCaptureController::NewDeviceClient() {
167 scoped_ptr
<media::VideoCaptureDevice::Client
> result(
168 new VideoCaptureDeviceClient(this->GetWeakPtr(), buffer_pool_
));
169 return result
.Pass();
172 void VideoCaptureController::AddClient(
173 const VideoCaptureControllerID
& id
,
174 VideoCaptureControllerEventHandler
* event_handler
,
175 base::ProcessHandle render_process
,
176 media::VideoCaptureSessionId session_id
,
177 const media::VideoCaptureParams
& params
) {
178 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
179 DVLOG(1) << "VideoCaptureController::AddClient, id " << id
.device_id
180 << ", " << params
.requested_format
.frame_size
.ToString()
181 << ", " << params
.requested_format
.frame_rate
182 << ", " << session_id
185 // If this is the first client added to the controller, cache the parameters.
186 if (!controller_clients_
.size())
187 video_capture_format_
= params
.requested_format
;
189 // Signal error in case device is already in error state.
190 if (state_
== VIDEO_CAPTURE_STATE_ERROR
) {
191 event_handler
->OnError(id
);
195 // Do nothing if this client has called AddClient before.
196 if (FindClient(id
, event_handler
, controller_clients_
))
199 ControllerClient
* client
= new ControllerClient(
200 id
, event_handler
, render_process
, session_id
, params
);
201 // If we already have gotten frame_info from the device, repeat it to the new
203 if (state_
== VIDEO_CAPTURE_STATE_STARTED
) {
204 controller_clients_
.push_back(client
);
209 int VideoCaptureController::RemoveClient(
210 const VideoCaptureControllerID
& id
,
211 VideoCaptureControllerEventHandler
* event_handler
) {
212 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
213 DVLOG(1) << "VideoCaptureController::RemoveClient, id " << id
.device_id
;
215 ControllerClient
* client
= FindClient(id
, event_handler
, controller_clients_
);
217 return kInvalidMediaCaptureSessionId
;
219 // Take back all buffers held by the |client|.
220 for (ControllerClient::ActiveBufferMap::iterator buffer_it
=
221 client
->active_buffers
.begin();
222 buffer_it
!= client
->active_buffers
.end();
224 buffer_pool_
->RelinquishConsumerHold(buffer_it
->first
, 1);
226 client
->active_buffers
.clear();
228 int session_id
= client
->session_id
;
229 controller_clients_
.remove(client
);
235 void VideoCaptureController::StopSession(int session_id
) {
236 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
237 DVLOG(1) << "VideoCaptureController::StopSession, id " << session_id
;
239 ControllerClient
* client
= FindClient(session_id
, controller_clients_
);
242 client
->session_closed
= true;
243 client
->event_handler
->OnEnded(client
->controller_id
);
247 void VideoCaptureController::ReturnBuffer(
248 const VideoCaptureControllerID
& id
,
249 VideoCaptureControllerEventHandler
* event_handler
,
251 const std::vector
<uint32
>& sync_points
) {
252 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
254 ControllerClient
* client
= FindClient(id
, event_handler
, controller_clients_
);
256 // If this buffer is not held by this client, or this client doesn't exist
257 // in controller, do nothing.
258 ControllerClient::ActiveBufferMap::iterator iter
;
259 if (!client
|| (iter
= client
->active_buffers
.find(buffer_id
)) ==
260 client
->active_buffers
.end()) {
264 scoped_refptr
<media::VideoFrame
> frame
= iter
->second
;
265 client
->active_buffers
.erase(iter
);
267 if (frame
->format() == media::VideoFrame::NATIVE_TEXTURE
) {
268 for (size_t i
= 0; i
< sync_points
.size(); i
++)
269 frame
->AppendReleaseSyncPoint(sync_points
[i
]);
272 buffer_pool_
->RelinquishConsumerHold(buffer_id
, 1);
275 const media::VideoCaptureFormat
&
276 VideoCaptureController::GetVideoCaptureFormat() const {
277 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
278 return video_capture_format_
;
281 scoped_refptr
<media::VideoCaptureDevice::Client::Buffer
>
282 VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer(
283 media::VideoFrame::Format format
,
284 const gfx::Size
& size
) {
285 return DoReserveOutputBuffer(format
, size
);
288 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedData(
291 const VideoCaptureFormat
& frame_format
,
293 base::TimeTicks timestamp
) {
294 TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedData");
296 if (!frame_format
.IsValid())
299 // Chopped pixels in width/height in case video capture device has odd
300 // numbers for width/height.
301 int chopped_width
= 0;
302 int chopped_height
= 0;
303 int new_unrotated_width
= frame_format
.frame_size
.width();
304 int new_unrotated_height
= frame_format
.frame_size
.height();
306 if (new_unrotated_width
& 1) {
307 --new_unrotated_width
;
310 if (new_unrotated_height
& 1) {
311 --new_unrotated_height
;
315 int destination_width
= new_unrotated_width
;
316 int destination_height
= new_unrotated_height
;
317 if (rotation
== 90 || rotation
== 270) {
318 destination_width
= new_unrotated_height
;
319 destination_height
= new_unrotated_width
;
321 const gfx::Size
dimensions(destination_width
, destination_height
);
322 if (!media::VideoFrame::IsValidConfig(media::VideoFrame::I420
,
324 gfx::Rect(dimensions
),
329 scoped_refptr
<Buffer
> buffer
=
330 DoReserveOutputBuffer(media::VideoFrame::I420
, dimensions
);
334 uint8
* yplane
= NULL
;
336 yplane
= reinterpret_cast<uint8
*>(buffer
->data());
339 media::VideoFrame::PlaneAllocationSize(
340 media::VideoFrame::I420
, media::VideoFrame::kYPlane
, dimensions
);
343 media::VideoFrame::PlaneAllocationSize(
344 media::VideoFrame::I420
, media::VideoFrame::kUPlane
, dimensions
);
345 int yplane_stride
= dimensions
.width();
346 int uv_plane_stride
= yplane_stride
/ 2;
349 libyuv::FourCC origin_colorspace
= libyuv::FOURCC_ANY
;
351 libyuv::RotationMode rotation_mode
= libyuv::kRotate0
;
353 rotation_mode
= libyuv::kRotate90
;
354 else if (rotation
== 180)
355 rotation_mode
= libyuv::kRotate180
;
356 else if (rotation
== 270)
357 rotation_mode
= libyuv::kRotate270
;
359 switch (frame_format
.pixel_format
) {
360 case media::PIXEL_FORMAT_UNKNOWN
: // Color format not set.
362 case media::PIXEL_FORMAT_I420
:
363 DCHECK(!chopped_width
&& !chopped_height
);
364 origin_colorspace
= libyuv::FOURCC_I420
;
366 case media::PIXEL_FORMAT_YV12
:
367 DCHECK(!chopped_width
&& !chopped_height
);
368 origin_colorspace
= libyuv::FOURCC_YV12
;
370 case media::PIXEL_FORMAT_NV21
:
371 DCHECK(!chopped_width
&& !chopped_height
);
372 origin_colorspace
= libyuv::FOURCC_NV21
;
374 case media::PIXEL_FORMAT_YUY2
:
375 DCHECK(!chopped_width
&& !chopped_height
);
376 origin_colorspace
= libyuv::FOURCC_YUY2
;
378 case media::PIXEL_FORMAT_UYVY
:
379 DCHECK(!chopped_width
&& !chopped_height
);
380 origin_colorspace
= libyuv::FOURCC_UYVY
;
382 case media::PIXEL_FORMAT_RGB24
:
383 origin_colorspace
= libyuv::FOURCC_24BG
;
385 // TODO(wjia): Currently, for RGB24 on WIN, capture device always
386 // passes in positive src_width and src_height. Remove this hardcoded
387 // value when nagative src_height is supported. The negative src_height
388 // indicates that vertical flipping is needed.
392 case media::PIXEL_FORMAT_ARGB
:
393 origin_colorspace
= libyuv::FOURCC_ARGB
;
395 case media::PIXEL_FORMAT_MJPEG
:
396 origin_colorspace
= libyuv::FOURCC_MJPG
;
402 libyuv::ConvertToI420(data
,
412 frame_format
.frame_size
.width(),
413 (flip
? -frame_format
.frame_size
.height() :
414 frame_format
.frame_size
.height()),
416 new_unrotated_height
,
419 scoped_refptr
<media::VideoFrame
> frame
=
420 media::VideoFrame::WrapExternalPackedMemory(
421 media::VideoFrame::I420
,
423 gfx::Rect(dimensions
),
426 media::VideoFrame::AllocationSize(media::VideoFrame::I420
,
428 base::SharedMemory::NULLHandle(),
433 VideoCaptureFormat
format(
434 dimensions
, frame_format
.frame_rate
, media::PIXEL_FORMAT_I420
);
435 BrowserThread::PostTask(
439 &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread
,
447 UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Width",
448 frame_format
.frame_size
.width());
449 UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Height",
450 frame_format
.frame_size
.height());
451 UMA_HISTOGRAM_ASPECT_RATIO("Media.VideoCapture.AspectRatio",
452 frame_format
.frame_size
.width(),
453 frame_format
.frame_size
.height());
454 UMA_HISTOGRAM_COUNTS("Media.VideoCapture.FrameRate",
455 frame_format
.frame_rate
);
456 UMA_HISTOGRAM_ENUMERATION("Media.VideoCapture.PixelFormat",
457 frame_format
.pixel_format
,
458 media::PIXEL_FORMAT_MAX
);
459 first_frame_
= false;
464 VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedVideoFrame(
465 const scoped_refptr
<Buffer
>& buffer
,
466 const VideoCaptureFormat
& buffer_format
,
467 const scoped_refptr
<media::VideoFrame
>& frame
,
468 base::TimeTicks timestamp
) {
469 BrowserThread::PostTask(
473 &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread
,
481 void VideoCaptureController::VideoCaptureDeviceClient::OnError(
482 const std::string
& reason
) {
483 MediaStreamManager::SendMessageToNativeLog(
484 "Error on video capture: " + reason
);
485 BrowserThread::PostTask(BrowserThread::IO
,
487 base::Bind(&VideoCaptureController::DoErrorOnIOThread
, controller_
));
490 scoped_refptr
<media::VideoCaptureDevice::Client::Buffer
>
491 VideoCaptureController::VideoCaptureDeviceClient::DoReserveOutputBuffer(
492 media::VideoFrame::Format format
,
493 const gfx::Size
& dimensions
) {
494 size_t frame_bytes
= 0;
495 if (format
== media::VideoFrame::NATIVE_TEXTURE
) {
496 DCHECK_EQ(dimensions
.width(), 0);
497 DCHECK_EQ(dimensions
.height(), 0);
499 // The capture pipeline expects I420 for now.
500 DCHECK_EQ(format
, media::VideoFrame::I420
)
501 << "Non-I420 output buffer format " << format
<< " requested";
502 frame_bytes
= media::VideoFrame::AllocationSize(format
, dimensions
);
505 int buffer_id_to_drop
= VideoCaptureBufferPool::kInvalidId
;
507 buffer_pool_
->ReserveForProducer(frame_bytes
, &buffer_id_to_drop
);
508 if (buffer_id
== VideoCaptureBufferPool::kInvalidId
)
512 buffer_pool_
->GetBufferInfo(buffer_id
, &data
, &size
);
514 scoped_refptr
<media::VideoCaptureDevice::Client::Buffer
> output_buffer(
515 new PoolBuffer(buffer_pool_
, buffer_id
, data
, size
));
517 if (buffer_id_to_drop
!= VideoCaptureBufferPool::kInvalidId
) {
518 BrowserThread::PostTask(BrowserThread::IO
,
520 base::Bind(&VideoCaptureController::DoBufferDestroyedOnIOThread
,
521 controller_
, buffer_id_to_drop
));
524 return output_buffer
;
527 VideoCaptureController::~VideoCaptureController() {
528 STLDeleteContainerPointers(controller_clients_
.begin(),
529 controller_clients_
.end());
532 void VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread(
533 const scoped_refptr
<media::VideoCaptureDevice::Client::Buffer
>& buffer
,
534 const media::VideoCaptureFormat
& buffer_format
,
535 const scoped_refptr
<media::VideoFrame
>& frame
,
536 base::TimeTicks timestamp
) {
537 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
538 DCHECK_NE(buffer
->id(), VideoCaptureBufferPool::kInvalidId
);
541 if (state_
== VIDEO_CAPTURE_STATE_STARTED
) {
542 for (ControllerClients::iterator client_it
= controller_clients_
.begin();
543 client_it
!= controller_clients_
.end(); ++client_it
) {
544 ControllerClient
* client
= *client_it
;
545 if (client
->session_closed
)
548 if (frame
->format() == media::VideoFrame::NATIVE_TEXTURE
) {
549 client
->event_handler
->OnMailboxBufferReady(client
->controller_id
,
551 *frame
->mailbox_holder(),
555 bool is_new_buffer
= client
->known_buffers
.insert(buffer
->id()).second
;
557 // On the first use of a buffer on a client, share the memory handle.
558 size_t memory_size
= 0;
559 base::SharedMemoryHandle remote_handle
= buffer_pool_
->ShareToProcess(
560 buffer
->id(), client
->render_process_handle
, &memory_size
);
561 client
->event_handler
->OnBufferCreated(
562 client
->controller_id
, remote_handle
, memory_size
, buffer
->id());
565 client
->event_handler
->OnBufferReady(
566 client
->controller_id
, buffer
->id(), buffer_format
, timestamp
);
570 client
->active_buffers
.insert(std::make_pair(buffer
->id(), frame
))
572 DCHECK(inserted
) << "Unexpected duplicate buffer: " << buffer
->id();
577 buffer_pool_
->HoldForConsumers(buffer
->id(), count
);
580 void VideoCaptureController::DoErrorOnIOThread() {
581 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
582 state_
= VIDEO_CAPTURE_STATE_ERROR
;
584 for (ControllerClients::iterator client_it
= controller_clients_
.begin();
585 client_it
!= controller_clients_
.end(); ++client_it
) {
586 ControllerClient
* client
= *client_it
;
587 if (client
->session_closed
)
590 client
->event_handler
->OnError(client
->controller_id
);
594 void VideoCaptureController::DoBufferDestroyedOnIOThread(
595 int buffer_id_to_drop
) {
596 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
598 for (ControllerClients::iterator client_it
= controller_clients_
.begin();
599 client_it
!= controller_clients_
.end(); ++client_it
) {
600 ControllerClient
* client
= *client_it
;
601 if (client
->session_closed
)
604 if (client
->known_buffers
.erase(buffer_id_to_drop
)) {
605 client
->event_handler
->OnBufferDestroyed(client
->controller_id
,
611 VideoCaptureController::ControllerClient
*
612 VideoCaptureController::FindClient(
613 const VideoCaptureControllerID
& id
,
614 VideoCaptureControllerEventHandler
* handler
,
615 const ControllerClients
& clients
) {
616 for (ControllerClients::const_iterator client_it
= clients
.begin();
617 client_it
!= clients
.end(); ++client_it
) {
618 if ((*client_it
)->controller_id
== id
&&
619 (*client_it
)->event_handler
== handler
) {
626 VideoCaptureController::ControllerClient
*
627 VideoCaptureController::FindClient(
629 const ControllerClients
& clients
) {
630 for (ControllerClients::const_iterator client_it
= clients
.begin();
631 client_it
!= clients
.end(); ++client_it
) {
632 if ((*client_it
)->session_id
== session_id
) {
639 int VideoCaptureController::GetClientCount() {
640 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
641 return controller_clients_
.size();
644 } // namespace content