1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/renderer_host/media/video_capture_controller.h"
10 #include "base/debug/trace_event.h"
11 #include "base/stl_util.h"
12 #include "content/browser/renderer_host/media/media_stream_manager.h"
13 #include "content/browser/renderer_host/media/video_capture_manager.h"
14 #include "content/public/browser/browser_thread.h"
15 #include "media/base/video_frame.h"
16 #include "media/base/video_util.h"
17 #include "media/base/yuv_convert.h"
19 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
20 #include "third_party/libyuv/include/libyuv.h"
23 using media::VideoCaptureFormat
;
29 // The number of buffers that VideoCaptureBufferPool should allocate.
30 const int kNoOfBuffers
= 3;
32 class PoolBuffer
: public media::VideoCaptureDevice::Client::Buffer
{
34 PoolBuffer(const scoped_refptr
<VideoCaptureBufferPool
>& pool
,
38 : Buffer(buffer_id
, data
, size
), pool_(pool
) {
43 virtual ~PoolBuffer() { pool_
->RelinquishProducerReservation(id()); }
45 const scoped_refptr
<VideoCaptureBufferPool
> pool_
;
48 } // anonymous namespace
50 struct VideoCaptureController::ControllerClient
{
51 ControllerClient(const VideoCaptureControllerID
& id
,
52 VideoCaptureControllerEventHandler
* handler
,
53 base::ProcessHandle render_process
,
54 media::VideoCaptureSessionId session_id
,
55 const media::VideoCaptureParams
& params
)
57 event_handler(handler
),
58 render_process_handle(render_process
),
59 session_id(session_id
),
61 session_closed(false) {}
63 ~ControllerClient() {}
65 // ID used for identifying this object.
66 const VideoCaptureControllerID controller_id
;
67 VideoCaptureControllerEventHandler
* const event_handler
;
69 // Handle to the render process that will receive the capture buffers.
70 const base::ProcessHandle render_process_handle
;
71 const media::VideoCaptureSessionId session_id
;
72 const media::VideoCaptureParams parameters
;
74 // Buffers that are currently known to this client.
75 std::set
<int> known_buffers
;
77 // Buffers currently held by this client.
78 std::set
<int> active_buffers
;
80 // State of capture session, controlled by VideoCaptureManager directly. This
81 // transitions to true as soon as StopSession() occurs, at which point the
82 // client is sent an OnEnded() event. However, because the client retains a
83 // VideoCaptureController* pointer, its ControllerClient entry lives on until
84 // it unregisters itself via RemoveClient(), which may happen asynchronously.
86 // TODO(nick): If we changed the semantics of VideoCaptureHost so that
87 // OnEnded() events were processed synchronously (with the RemoveClient() done
88 // implicitly), we could avoid tracking this state here in the Controller, and
89 // simplify the code in both places.
93 // Receives events from the VideoCaptureDevice and posts them to a
94 // VideoCaptureController on the IO thread. An instance of this class may safely
95 // outlive its target VideoCaptureController.
97 // Methods of this class may be called from any thread, and in practice will
98 // often be called on some auxiliary thread depending on the platform and the
99 // device type; including, for example, the DirectShow thread on Windows, the
100 // v4l2_thread on Linux, and the UI thread for tab capture.
101 class VideoCaptureController::VideoCaptureDeviceClient
102 : public media::VideoCaptureDevice::Client
{
104 explicit VideoCaptureDeviceClient(
105 const base::WeakPtr
<VideoCaptureController
>& controller
,
106 const scoped_refptr
<VideoCaptureBufferPool
>& buffer_pool
);
107 virtual ~VideoCaptureDeviceClient();
109 // VideoCaptureDevice::Client implementation.
110 virtual scoped_refptr
<Buffer
> ReserveOutputBuffer(
111 media::VideoFrame::Format format
,
112 const gfx::Size
& size
) OVERRIDE
;
113 virtual void OnIncomingCapturedFrame(const uint8
* data
,
115 base::Time timestamp
,
119 const VideoCaptureFormat
& frame_format
)
121 virtual void OnIncomingCapturedBuffer(const scoped_refptr
<Buffer
>& buffer
,
122 media::VideoFrame::Format format
,
123 const gfx::Size
& dimensions
,
124 base::Time timestamp
,
125 int frame_rate
) OVERRIDE
;
126 virtual void OnError() OVERRIDE
;
129 scoped_refptr
<Buffer
> DoReserveOutputBuffer(media::VideoFrame::Format format
,
130 const gfx::Size
& dimensions
,
133 // The controller to which we post events.
134 const base::WeakPtr
<VideoCaptureController
> controller_
;
136 // The pool of shared-memory buffers used for capturing.
137 const scoped_refptr
<VideoCaptureBufferPool
> buffer_pool_
;
139 // The set of buffers that have been used for rotated capturing.
140 std::set
<int> rotated_buffers_
;
143 VideoCaptureController::VideoCaptureController()
144 : buffer_pool_(new VideoCaptureBufferPool(kNoOfBuffers
)),
145 state_(VIDEO_CAPTURE_STATE_STARTED
),
146 weak_ptr_factory_(this) {
149 VideoCaptureController::VideoCaptureDeviceClient::VideoCaptureDeviceClient(
150 const base::WeakPtr
<VideoCaptureController
>& controller
,
151 const scoped_refptr
<VideoCaptureBufferPool
>& buffer_pool
)
152 : controller_(controller
), buffer_pool_(buffer_pool
) {}
154 VideoCaptureController::VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {}
156 base::WeakPtr
<VideoCaptureController
> VideoCaptureController::GetWeakPtr() {
157 return weak_ptr_factory_
.GetWeakPtr();
160 scoped_ptr
<media::VideoCaptureDevice::Client
>
161 VideoCaptureController::NewDeviceClient() {
162 scoped_ptr
<media::VideoCaptureDevice::Client
> result(
163 new VideoCaptureDeviceClient(this->GetWeakPtr(), buffer_pool_
));
164 return result
.Pass();
167 void VideoCaptureController::AddClient(
168 const VideoCaptureControllerID
& id
,
169 VideoCaptureControllerEventHandler
* event_handler
,
170 base::ProcessHandle render_process
,
171 media::VideoCaptureSessionId session_id
,
172 const media::VideoCaptureParams
& params
) {
173 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
174 DVLOG(1) << "VideoCaptureController::AddClient, id " << id
.device_id
175 << ", " << params
.requested_format
.frame_size
.ToString()
176 << ", " << params
.requested_format
.frame_rate
177 << ", " << session_id
180 // Signal error in case device is already in error state.
181 if (state_
== VIDEO_CAPTURE_STATE_ERROR
) {
182 event_handler
->OnError(id
);
186 // Do nothing if this client has called AddClient before.
187 if (FindClient(id
, event_handler
, controller_clients_
))
190 ControllerClient
* client
= new ControllerClient(
191 id
, event_handler
, render_process
, session_id
, params
);
192 // If we already have gotten frame_info from the device, repeat it to the new
194 if (state_
== VIDEO_CAPTURE_STATE_STARTED
) {
195 controller_clients_
.push_back(client
);
200 int VideoCaptureController::RemoveClient(
201 const VideoCaptureControllerID
& id
,
202 VideoCaptureControllerEventHandler
* event_handler
) {
203 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
204 DVLOG(1) << "VideoCaptureController::RemoveClient, id " << id
.device_id
;
206 ControllerClient
* client
= FindClient(id
, event_handler
, controller_clients_
);
208 return kInvalidMediaCaptureSessionId
;
210 // Take back all buffers held by the |client|.
211 for (std::set
<int>::iterator buffer_it
= client
->active_buffers
.begin();
212 buffer_it
!= client
->active_buffers
.end();
214 int buffer_id
= *buffer_it
;
215 buffer_pool_
->RelinquishConsumerHold(buffer_id
, 1);
217 client
->active_buffers
.clear();
219 int session_id
= client
->session_id
;
220 controller_clients_
.remove(client
);
226 void VideoCaptureController::StopSession(int session_id
) {
227 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
228 DVLOG(1) << "VideoCaptureController::StopSession, id " << session_id
;
230 ControllerClient
* client
= FindClient(session_id
, controller_clients_
);
233 client
->session_closed
= true;
234 client
->event_handler
->OnEnded(client
->controller_id
);
238 void VideoCaptureController::ReturnBuffer(
239 const VideoCaptureControllerID
& id
,
240 VideoCaptureControllerEventHandler
* event_handler
,
242 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
244 ControllerClient
* client
= FindClient(id
, event_handler
, controller_clients_
);
246 // If this buffer is not held by this client, or this client doesn't exist
247 // in controller, do nothing.
248 if (!client
|| !client
->active_buffers
.erase(buffer_id
)) {
253 buffer_pool_
->RelinquishConsumerHold(buffer_id
, 1);
256 scoped_refptr
<media::VideoCaptureDevice::Client::Buffer
>
257 VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer(
258 media::VideoFrame::Format format
,
259 const gfx::Size
& size
) {
260 return DoReserveOutputBuffer(format
, size
, 0);
263 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame(
266 base::Time timestamp
,
270 const VideoCaptureFormat
& frame_format
) {
271 TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame");
273 if (!frame_format
.IsValid())
276 // Chopped pixels in width/height in case video capture device has odd
277 // numbers for width/height.
278 int chopped_width
= 0;
279 int chopped_height
= 0;
280 int new_width
= frame_format
.frame_size
.width();
281 int new_height
= frame_format
.frame_size
.height();
287 if (new_height
& 1) {
292 const gfx::Size
dimensions(new_width
, new_height
);
293 scoped_refptr
<Buffer
> buffer
=
294 DoReserveOutputBuffer(media::VideoFrame::I420
, dimensions
, rotation
);
298 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
299 uint8
* yplane
= reinterpret_cast<uint8
*>(buffer
->data());
302 media::VideoFrame::PlaneAllocationSize(
303 media::VideoFrame::I420
, media::VideoFrame::kYPlane
, dimensions
);
306 media::VideoFrame::PlaneAllocationSize(
307 media::VideoFrame::I420
, media::VideoFrame::kUPlane
, dimensions
);
308 int yplane_stride
= new_width
;
309 int uv_plane_stride
= new_width
/ 2;
312 int destination_width
= new_width
;
313 int destination_height
= new_height
;
314 libyuv::FourCC origin_colorspace
= libyuv::FOURCC_ANY
;
316 // When rotating by 90 and 270 degrees swap |flip_horiz| and |flip_vert|
317 // because ConvertToI420() flips image before rotation, while
318 // OnIncomingCapturedFrame() interface assumes that rotation happens before
320 if (rotation
== 90 || rotation
== 270)
321 std::swap(flip_horiz
, flip_vert
);
323 // Assuming rotation happens first and flips next, we can consolidate both
324 // vertical and horizontal flips together with rotation into two variables:
325 // new_rotation = (rotation + 180 * horizontal_flip) modulo 360
326 // new_vertical_flip = horizontal_flip XOR vertical_flip
327 int new_rotation_angle
= (rotation
+ 180 * flip_horiz
) % 360;
328 libyuv::RotationMode rotation_mode
= libyuv::kRotate0
;
329 if (new_rotation_angle
== 90)
330 rotation_mode
= libyuv::kRotate90
;
331 else if (new_rotation_angle
== 180)
332 rotation_mode
= libyuv::kRotate180
;
333 else if (new_rotation_angle
== 270)
334 rotation_mode
= libyuv::kRotate270
;
336 switch (frame_format
.pixel_format
) {
337 case media::PIXEL_FORMAT_UNKNOWN
: // Color format not set.
339 case media::PIXEL_FORMAT_I420
:
340 DCHECK(!chopped_width
&& !chopped_height
);
341 origin_colorspace
= libyuv::FOURCC_I420
;
343 case media::PIXEL_FORMAT_YV12
:
344 DCHECK(!chopped_width
&& !chopped_height
);
345 origin_colorspace
= libyuv::FOURCC_YV12
;
347 case media::PIXEL_FORMAT_NV21
:
348 DCHECK(!chopped_width
&& !chopped_height
);
349 origin_colorspace
= libyuv::FOURCC_NV21
;
351 case media::PIXEL_FORMAT_YUY2
:
352 DCHECK(!chopped_width
&& !chopped_height
);
353 origin_colorspace
= libyuv::FOURCC_YUY2
;
355 case media::PIXEL_FORMAT_UYVY
:
356 DCHECK(!chopped_width
&& !chopped_height
);
357 origin_colorspace
= libyuv::FOURCC_UYVY
;
359 case media::PIXEL_FORMAT_RGB24
:
360 origin_colorspace
= libyuv::FOURCC_RAW
;
362 case media::PIXEL_FORMAT_ARGB
:
363 origin_colorspace
= libyuv::FOURCC_ARGB
;
365 case media::PIXEL_FORMAT_MJPEG
:
366 origin_colorspace
= libyuv::FOURCC_MJPG
;
372 int need_convert_rgb24_on_win
= false;
374 // kRGB24 on Windows start at the bottom line and has a negative stride. This
375 // is not supported by libyuv, so the media API is used instead.
376 if (frame_format
.pixel_format
== media::PIXEL_FORMAT_RGB24
) {
377 // Rotation and flipping is not supported in kRGB24 and OS_WIN case.
378 DCHECK(!rotation
&& !flip_vert
&& !flip_horiz
);
379 need_convert_rgb24_on_win
= true;
382 if (need_convert_rgb24_on_win
) {
383 int rgb_stride
= -3 * (new_width
+ chopped_width
);
384 const uint8
* rgb_src
= data
+ 3 * (new_width
+ chopped_width
) *
385 (new_height
- 1 + chopped_height
);
386 media::ConvertRGB24ToYUV(rgb_src
,
396 if (new_rotation_angle
==90 || new_rotation_angle
==270){
397 // To be compatible with non-libyuv code in RotatePlaneByPixels, when
398 // rotating by 90/270, only the maximum square portion located in the
399 // center of the image is rotated. F.i. 640x480 pixels, only the central
400 // 480 pixels would be rotated and the leftmost and rightmost 80 columns
401 // would be ignored. This process is called letterboxing.
402 int letterbox_thickness
= abs(new_width
- new_height
) / 2;
403 if (destination_width
> destination_height
) {
404 yplane
+= letterbox_thickness
;
405 uplane
+= letterbox_thickness
/ 2;
406 vplane
+= letterbox_thickness
/ 2;
407 destination_width
= destination_height
;
409 yplane
+= letterbox_thickness
* destination_width
;
410 uplane
+= (letterbox_thickness
* destination_width
) / 2;
411 vplane
+= (letterbox_thickness
* destination_width
) / 2;
412 destination_height
= destination_width
;
415 libyuv::ConvertToI420(data
,
425 new_width
+ chopped_width
,
426 new_height
* (flip_vert
^ flip_horiz
? -1 : 1),
433 // Libyuv is not linked in for Android WebView builds, but video capture is
434 // not used in those builds either. Whenever libyuv is added in that build,
435 // address all these #ifdef parts, see http://crbug.com/299611 .
437 #endif // if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
438 BrowserThread::PostTask(
442 &VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread
,
446 frame_format
.frame_rate
,
450 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedBuffer(
451 const scoped_refptr
<Buffer
>& buffer
,
452 media::VideoFrame::Format format
,
453 const gfx::Size
& dimensions
,
454 base::Time timestamp
,
456 // The capture pipeline expects I420 for now.
457 DCHECK_EQ(format
, media::VideoFrame::I420
)
458 << "Non-I420 output buffer returned";
460 BrowserThread::PostTask(
464 &VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread
,
472 void VideoCaptureController::VideoCaptureDeviceClient::OnError() {
473 BrowserThread::PostTask(BrowserThread::IO
,
475 base::Bind(&VideoCaptureController::DoErrorOnIOThread
, controller_
));
478 scoped_refptr
<media::VideoCaptureDevice::Client::Buffer
>
479 VideoCaptureController::VideoCaptureDeviceClient::DoReserveOutputBuffer(
480 media::VideoFrame::Format format
,
481 const gfx::Size
& dimensions
,
483 // The capture pipeline expects I420 for now.
484 DCHECK_EQ(format
, media::VideoFrame::I420
)
485 << "Non-I420 output buffer requested";
487 int buffer_id_to_drop
= VideoCaptureBufferPool::kInvalidId
;
488 const size_t frame_bytes
=
489 media::VideoFrame::AllocationSize(format
, dimensions
);
492 buffer_pool_
->ReserveForProducer(frame_bytes
, &buffer_id_to_drop
);
493 if (buffer_id
== VideoCaptureBufferPool::kInvalidId
)
497 buffer_pool_
->GetBufferInfo(buffer_id
, &data
, &size
);
499 scoped_refptr
<media::VideoCaptureDevice::Client::Buffer
> output_buffer(
500 new PoolBuffer(buffer_pool_
, buffer_id
, data
, size
));
502 if (buffer_id_to_drop
!= VideoCaptureBufferPool::kInvalidId
) {
503 BrowserThread::PostTask(BrowserThread::IO
,
505 base::Bind(&VideoCaptureController::DoBufferDestroyedOnIOThread
,
506 controller_
, buffer_id_to_drop
));
507 rotated_buffers_
.erase(buffer_id_to_drop
);
510 // If a 90/270 rotation is required, letterboxing will be required. If the
511 // returned frame has not been rotated before, then the letterbox borders will
512 // not yet have been cleared and we should clear them now.
513 if ((rotation
% 180) == 0) {
514 rotated_buffers_
.erase(buffer_id
);
516 if (rotated_buffers_
.insert(buffer_id
).second
) {
517 scoped_refptr
<media::VideoFrame
> frame
=
518 media::VideoFrame::WrapExternalPackedMemory(
519 media::VideoFrame::I420
,
521 gfx::Rect(dimensions
),
523 static_cast<uint8
*>(output_buffer
->data()),
524 output_buffer
->size(),
525 base::SharedMemory::NULLHandle(),
528 media::FillYUV(frame
, 0, 128, 128);
532 return output_buffer
;
535 VideoCaptureController::~VideoCaptureController() {
536 STLDeleteContainerPointers(controller_clients_
.begin(),
537 controller_clients_
.end());
540 void VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread(
541 scoped_refptr
<media::VideoCaptureDevice::Client::Buffer
> buffer
,
542 const gfx::Size
& dimensions
,
544 base::Time timestamp
) {
545 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
546 DCHECK_NE(buffer
->id(), VideoCaptureBufferPool::kInvalidId
);
548 VideoCaptureFormat
frame_format(
549 dimensions
, frame_rate
, media::PIXEL_FORMAT_I420
);
552 if (state_
== VIDEO_CAPTURE_STATE_STARTED
) {
553 for (ControllerClients::iterator client_it
= controller_clients_
.begin();
554 client_it
!= controller_clients_
.end(); ++client_it
) {
555 ControllerClient
* client
= *client_it
;
556 if (client
->session_closed
)
559 bool is_new_buffer
= client
->known_buffers
.insert(buffer
->id()).second
;
561 // On the first use of a buffer on a client, share the memory handle.
562 size_t memory_size
= 0;
563 base::SharedMemoryHandle remote_handle
= buffer_pool_
->ShareToProcess(
564 buffer
->id(), client
->render_process_handle
, &memory_size
);
565 client
->event_handler
->OnBufferCreated(
566 client
->controller_id
, remote_handle
, memory_size
, buffer
->id());
569 client
->event_handler
->OnBufferReady(
570 client
->controller_id
, buffer
->id(), timestamp
, frame_format
);
571 bool inserted
= client
->active_buffers
.insert(buffer
->id()).second
;
572 DCHECK(inserted
) << "Unexpected duplicate buffer: " << buffer
->id();
577 buffer_pool_
->HoldForConsumers(buffer
->id(), count
);
580 void VideoCaptureController::DoErrorOnIOThread() {
581 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
582 state_
= VIDEO_CAPTURE_STATE_ERROR
;
584 for (ControllerClients::iterator client_it
= controller_clients_
.begin();
585 client_it
!= controller_clients_
.end(); ++client_it
) {
586 ControllerClient
* client
= *client_it
;
587 if (client
->session_closed
)
590 client
->event_handler
->OnError(client
->controller_id
);
594 void VideoCaptureController::DoBufferDestroyedOnIOThread(
595 int buffer_id_to_drop
) {
596 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
598 for (ControllerClients::iterator client_it
= controller_clients_
.begin();
599 client_it
!= controller_clients_
.end(); ++client_it
) {
600 ControllerClient
* client
= *client_it
;
601 if (client
->session_closed
)
604 if (client
->known_buffers
.erase(buffer_id_to_drop
)) {
605 client
->event_handler
->OnBufferDestroyed(client
->controller_id
,
611 VideoCaptureController::ControllerClient
*
612 VideoCaptureController::FindClient(
613 const VideoCaptureControllerID
& id
,
614 VideoCaptureControllerEventHandler
* handler
,
615 const ControllerClients
& clients
) {
616 for (ControllerClients::const_iterator client_it
= clients
.begin();
617 client_it
!= clients
.end(); ++client_it
) {
618 if ((*client_it
)->controller_id
== id
&&
619 (*client_it
)->event_handler
== handler
) {
626 VideoCaptureController::ControllerClient
*
627 VideoCaptureController::FindClient(
629 const ControllerClients
& clients
) {
630 for (ControllerClients::const_iterator client_it
= clients
.begin();
631 client_it
!= clients
.end(); ++client_it
) {
632 if ((*client_it
)->session_id
== session_id
) {
639 int VideoCaptureController::GetClientCount() {
640 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
641 return controller_clients_
.size();
644 } // namespace content