Ignore non-active fullscreen windows for shelf state.
[chromium-blink-merge.git] / content / browser / renderer_host / media / video_capture_controller.cc
blob7581df39d59d32d3a0972b4c32d6c8782e956197
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/renderer_host/media/video_capture_controller.h"
7 #include <set>
9 #include "base/bind.h"
10 #include "base/debug/trace_event.h"
11 #include "base/stl_util.h"
12 #include "content/browser/renderer_host/media/media_stream_manager.h"
13 #include "content/browser/renderer_host/media/video_capture_manager.h"
14 #include "content/public/browser/browser_thread.h"
15 #include "media/base/video_frame.h"
16 #include "media/base/video_util.h"
17 #include "media/base/yuv_convert.h"
19 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
20 #include "third_party/libyuv/include/libyuv.h"
21 #endif
23 using media::VideoCaptureFormat;
25 namespace content {
27 namespace {
29 // The number of buffers that VideoCaptureBufferPool should allocate.
30 const int kNoOfBuffers = 3;
32 class PoolBuffer : public media::VideoCaptureDevice::Client::Buffer {
33 public:
34 PoolBuffer(const scoped_refptr<VideoCaptureBufferPool>& pool,
35 int buffer_id,
36 void* data,
37 size_t size)
38 : Buffer(buffer_id, data, size), pool_(pool) {
39 DCHECK(pool_);
42 private:
43 virtual ~PoolBuffer() { pool_->RelinquishProducerReservation(id()); }
45 const scoped_refptr<VideoCaptureBufferPool> pool_;
48 } // anonymous namespace
50 struct VideoCaptureController::ControllerClient {
51 ControllerClient(const VideoCaptureControllerID& id,
52 VideoCaptureControllerEventHandler* handler,
53 base::ProcessHandle render_process,
54 media::VideoCaptureSessionId session_id,
55 const media::VideoCaptureParams& params)
56 : controller_id(id),
57 event_handler(handler),
58 render_process_handle(render_process),
59 session_id(session_id),
60 parameters(params),
61 session_closed(false) {}
63 ~ControllerClient() {}
65 // ID used for identifying this object.
66 const VideoCaptureControllerID controller_id;
67 VideoCaptureControllerEventHandler* const event_handler;
69 // Handle to the render process that will receive the capture buffers.
70 const base::ProcessHandle render_process_handle;
71 const media::VideoCaptureSessionId session_id;
72 const media::VideoCaptureParams parameters;
74 // Buffers that are currently known to this client.
75 std::set<int> known_buffers;
77 // Buffers currently held by this client.
78 std::set<int> active_buffers;
80 // State of capture session, controlled by VideoCaptureManager directly. This
81 // transitions to true as soon as StopSession() occurs, at which point the
82 // client is sent an OnEnded() event. However, because the client retains a
83 // VideoCaptureController* pointer, its ControllerClient entry lives on until
84 // it unregisters itself via RemoveClient(), which may happen asynchronously.
86 // TODO(nick): If we changed the semantics of VideoCaptureHost so that
87 // OnEnded() events were processed synchronously (with the RemoveClient() done
88 // implicitly), we could avoid tracking this state here in the Controller, and
89 // simplify the code in both places.
90 bool session_closed;
93 // Receives events from the VideoCaptureDevice and posts them to a
94 // VideoCaptureController on the IO thread. An instance of this class may safely
95 // outlive its target VideoCaptureController.
97 // Methods of this class may be called from any thread, and in practice will
98 // often be called on some auxiliary thread depending on the platform and the
99 // device type; including, for example, the DirectShow thread on Windows, the
100 // v4l2_thread on Linux, and the UI thread for tab capture.
101 class VideoCaptureController::VideoCaptureDeviceClient
102 : public media::VideoCaptureDevice::Client {
103 public:
104 explicit VideoCaptureDeviceClient(
105 const base::WeakPtr<VideoCaptureController>& controller,
106 const scoped_refptr<VideoCaptureBufferPool>& buffer_pool);
107 virtual ~VideoCaptureDeviceClient();
109 // VideoCaptureDevice::Client implementation.
110 virtual scoped_refptr<Buffer> ReserveOutputBuffer(
111 media::VideoFrame::Format format,
112 const gfx::Size& size) OVERRIDE;
113 virtual void OnIncomingCapturedFrame(const uint8* data,
114 int length,
115 base::Time timestamp,
116 int rotation,
117 bool flip_vert,
118 bool flip_horiz,
119 const VideoCaptureFormat& frame_format)
120 OVERRIDE;
121 virtual void OnIncomingCapturedBuffer(const scoped_refptr<Buffer>& buffer,
122 media::VideoFrame::Format format,
123 const gfx::Size& dimensions,
124 base::Time timestamp,
125 int frame_rate) OVERRIDE;
126 virtual void OnError() OVERRIDE;
128 private:
129 scoped_refptr<Buffer> DoReserveOutputBuffer(media::VideoFrame::Format format,
130 const gfx::Size& dimensions,
131 int rotation);
133 // The controller to which we post events.
134 const base::WeakPtr<VideoCaptureController> controller_;
136 // The pool of shared-memory buffers used for capturing.
137 const scoped_refptr<VideoCaptureBufferPool> buffer_pool_;
139 // The set of buffers that have been used for rotated capturing.
140 std::set<int> rotated_buffers_;
143 VideoCaptureController::VideoCaptureController()
144 : buffer_pool_(new VideoCaptureBufferPool(kNoOfBuffers)),
145 state_(VIDEO_CAPTURE_STATE_STARTED),
146 weak_ptr_factory_(this) {
149 VideoCaptureController::VideoCaptureDeviceClient::VideoCaptureDeviceClient(
150 const base::WeakPtr<VideoCaptureController>& controller,
151 const scoped_refptr<VideoCaptureBufferPool>& buffer_pool)
152 : controller_(controller), buffer_pool_(buffer_pool) {}
154 VideoCaptureController::VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {}
156 base::WeakPtr<VideoCaptureController> VideoCaptureController::GetWeakPtr() {
157 return weak_ptr_factory_.GetWeakPtr();
160 scoped_ptr<media::VideoCaptureDevice::Client>
161 VideoCaptureController::NewDeviceClient() {
162 scoped_ptr<media::VideoCaptureDevice::Client> result(
163 new VideoCaptureDeviceClient(this->GetWeakPtr(), buffer_pool_));
164 return result.Pass();
167 void VideoCaptureController::AddClient(
168 const VideoCaptureControllerID& id,
169 VideoCaptureControllerEventHandler* event_handler,
170 base::ProcessHandle render_process,
171 media::VideoCaptureSessionId session_id,
172 const media::VideoCaptureParams& params) {
173 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
174 DVLOG(1) << "VideoCaptureController::AddClient, id " << id.device_id
175 << ", " << params.requested_format.frame_size.ToString()
176 << ", " << params.requested_format.frame_rate
177 << ", " << session_id
178 << ")";
180 // Signal error in case device is already in error state.
181 if (state_ == VIDEO_CAPTURE_STATE_ERROR) {
182 event_handler->OnError(id);
183 return;
186 // Do nothing if this client has called AddClient before.
187 if (FindClient(id, event_handler, controller_clients_))
188 return;
190 ControllerClient* client = new ControllerClient(
191 id, event_handler, render_process, session_id, params);
192 // If we already have gotten frame_info from the device, repeat it to the new
193 // client.
194 if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
195 controller_clients_.push_back(client);
196 return;
200 int VideoCaptureController::RemoveClient(
201 const VideoCaptureControllerID& id,
202 VideoCaptureControllerEventHandler* event_handler) {
203 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
204 DVLOG(1) << "VideoCaptureController::RemoveClient, id " << id.device_id;
206 ControllerClient* client = FindClient(id, event_handler, controller_clients_);
207 if (!client)
208 return kInvalidMediaCaptureSessionId;
210 // Take back all buffers held by the |client|.
211 for (std::set<int>::iterator buffer_it = client->active_buffers.begin();
212 buffer_it != client->active_buffers.end();
213 ++buffer_it) {
214 int buffer_id = *buffer_it;
215 buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
217 client->active_buffers.clear();
219 int session_id = client->session_id;
220 controller_clients_.remove(client);
221 delete client;
223 return session_id;
226 void VideoCaptureController::StopSession(int session_id) {
227 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
228 DVLOG(1) << "VideoCaptureController::StopSession, id " << session_id;
230 ControllerClient* client = FindClient(session_id, controller_clients_);
232 if (client) {
233 client->session_closed = true;
234 client->event_handler->OnEnded(client->controller_id);
238 void VideoCaptureController::ReturnBuffer(
239 const VideoCaptureControllerID& id,
240 VideoCaptureControllerEventHandler* event_handler,
241 int buffer_id) {
242 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
244 ControllerClient* client = FindClient(id, event_handler, controller_clients_);
246 // If this buffer is not held by this client, or this client doesn't exist
247 // in controller, do nothing.
248 if (!client || !client->active_buffers.erase(buffer_id)) {
249 NOTREACHED();
250 return;
253 buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
256 scoped_refptr<media::VideoCaptureDevice::Client::Buffer>
257 VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer(
258 media::VideoFrame::Format format,
259 const gfx::Size& size) {
260 return DoReserveOutputBuffer(format, size, 0);
263 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame(
264 const uint8* data,
265 int length,
266 base::Time timestamp,
267 int rotation,
268 bool flip_vert,
269 bool flip_horiz,
270 const VideoCaptureFormat& frame_format) {
271 TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame");
273 if (!frame_format.IsValid())
274 return;
276 // Chopped pixels in width/height in case video capture device has odd
277 // numbers for width/height.
278 int chopped_width = 0;
279 int chopped_height = 0;
280 int new_width = frame_format.frame_size.width();
281 int new_height = frame_format.frame_size.height();
283 if (new_width & 1) {
284 --new_width;
285 chopped_width = 1;
287 if (new_height & 1) {
288 --new_height;
289 chopped_height = 1;
292 const gfx::Size dimensions(new_width, new_height);
293 scoped_refptr<Buffer> buffer =
294 DoReserveOutputBuffer(media::VideoFrame::I420, dimensions, rotation);
296 if (!buffer)
297 return;
298 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
299 uint8* yplane = reinterpret_cast<uint8*>(buffer->data());
300 uint8* uplane =
301 yplane +
302 media::VideoFrame::PlaneAllocationSize(
303 media::VideoFrame::I420, media::VideoFrame::kYPlane, dimensions);
304 uint8* vplane =
305 uplane +
306 media::VideoFrame::PlaneAllocationSize(
307 media::VideoFrame::I420, media::VideoFrame::kUPlane, dimensions);
308 int yplane_stride = new_width;
309 int uv_plane_stride = new_width / 2;
310 int crop_x = 0;
311 int crop_y = 0;
312 int destination_width = new_width;
313 int destination_height = new_height;
314 libyuv::FourCC origin_colorspace = libyuv::FOURCC_ANY;
316 // When rotating by 90 and 270 degrees swap |flip_horiz| and |flip_vert|
317 // because ConvertToI420() flips image before rotation, while
318 // OnIncomingCapturedFrame() interface assumes that rotation happens before
319 // flips.
320 if (rotation == 90 || rotation == 270)
321 std::swap(flip_horiz, flip_vert);
323 // Assuming rotation happens first and flips next, we can consolidate both
324 // vertical and horizontal flips together with rotation into two variables:
325 // new_rotation = (rotation + 180 * horizontal_flip) modulo 360
326 // new_vertical_flip = horizontal_flip XOR vertical_flip
327 int new_rotation_angle = (rotation + 180 * flip_horiz) % 360;
328 libyuv::RotationMode rotation_mode = libyuv::kRotate0;
329 if (new_rotation_angle == 90)
330 rotation_mode = libyuv::kRotate90;
331 else if (new_rotation_angle == 180)
332 rotation_mode = libyuv::kRotate180;
333 else if (new_rotation_angle == 270)
334 rotation_mode = libyuv::kRotate270;
336 switch (frame_format.pixel_format) {
337 case media::PIXEL_FORMAT_UNKNOWN: // Color format not set.
338 break;
339 case media::PIXEL_FORMAT_I420:
340 DCHECK(!chopped_width && !chopped_height);
341 origin_colorspace = libyuv::FOURCC_I420;
342 break;
343 case media::PIXEL_FORMAT_YV12:
344 DCHECK(!chopped_width && !chopped_height);
345 origin_colorspace = libyuv::FOURCC_YV12;
346 break;
347 case media::PIXEL_FORMAT_NV21:
348 DCHECK(!chopped_width && !chopped_height);
349 origin_colorspace = libyuv::FOURCC_NV21;
350 break;
351 case media::PIXEL_FORMAT_YUY2:
352 DCHECK(!chopped_width && !chopped_height);
353 origin_colorspace = libyuv::FOURCC_YUY2;
354 break;
355 case media::PIXEL_FORMAT_UYVY:
356 DCHECK(!chopped_width && !chopped_height);
357 origin_colorspace = libyuv::FOURCC_UYVY;
358 break;
359 case media::PIXEL_FORMAT_RGB24:
360 origin_colorspace = libyuv::FOURCC_RAW;
361 break;
362 case media::PIXEL_FORMAT_ARGB:
363 origin_colorspace = libyuv::FOURCC_ARGB;
364 break;
365 case media::PIXEL_FORMAT_MJPEG:
366 origin_colorspace = libyuv::FOURCC_MJPG;
367 break;
368 default:
369 NOTREACHED();
372 int need_convert_rgb24_on_win = false;
373 #if defined(OS_WIN)
374 // kRGB24 on Windows start at the bottom line and has a negative stride. This
375 // is not supported by libyuv, so the media API is used instead.
376 if (frame_format.pixel_format == media::PIXEL_FORMAT_RGB24) {
377 // Rotation and flipping is not supported in kRGB24 and OS_WIN case.
378 DCHECK(!rotation && !flip_vert && !flip_horiz);
379 need_convert_rgb24_on_win = true;
381 #endif
382 if (need_convert_rgb24_on_win) {
383 int rgb_stride = -3 * (new_width + chopped_width);
384 const uint8* rgb_src = data + 3 * (new_width + chopped_width) *
385 (new_height - 1 + chopped_height);
386 media::ConvertRGB24ToYUV(rgb_src,
387 yplane,
388 uplane,
389 vplane,
390 new_width,
391 new_height,
392 rgb_stride,
393 yplane_stride,
394 uv_plane_stride);
395 } else {
396 if (new_rotation_angle==90 || new_rotation_angle==270){
397 // To be compatible with non-libyuv code in RotatePlaneByPixels, when
398 // rotating by 90/270, only the maximum square portion located in the
399 // center of the image is rotated. F.i. 640x480 pixels, only the central
400 // 480 pixels would be rotated and the leftmost and rightmost 80 columns
401 // would be ignored. This process is called letterboxing.
402 int letterbox_thickness = abs(new_width - new_height) / 2;
403 if (destination_width > destination_height) {
404 yplane += letterbox_thickness;
405 uplane += letterbox_thickness / 2;
406 vplane += letterbox_thickness / 2;
407 destination_width = destination_height;
408 } else {
409 yplane += letterbox_thickness * destination_width;
410 uplane += (letterbox_thickness * destination_width) / 2;
411 vplane += (letterbox_thickness * destination_width) / 2;
412 destination_height = destination_width;
415 libyuv::ConvertToI420(data,
416 length,
417 yplane,
418 yplane_stride,
419 uplane,
420 uv_plane_stride,
421 vplane,
422 uv_plane_stride,
423 crop_x,
424 crop_y,
425 new_width + chopped_width,
426 new_height * (flip_vert ^ flip_horiz ? -1 : 1),
427 destination_width,
428 destination_height,
429 rotation_mode,
430 origin_colorspace);
432 #else
433 // Libyuv is not linked in for Android WebView builds, but video capture is
434 // not used in those builds either. Whenever libyuv is added in that build,
435 // address all these #ifdef parts, see http://crbug.com/299611 .
436 NOTREACHED();
437 #endif // if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
438 BrowserThread::PostTask(
439 BrowserThread::IO,
440 FROM_HERE,
441 base::Bind(
442 &VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread,
443 controller_,
444 buffer,
445 dimensions,
446 frame_format.frame_rate,
447 timestamp));
450 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedBuffer(
451 const scoped_refptr<Buffer>& buffer,
452 media::VideoFrame::Format format,
453 const gfx::Size& dimensions,
454 base::Time timestamp,
455 int frame_rate) {
456 // The capture pipeline expects I420 for now.
457 DCHECK_EQ(format, media::VideoFrame::I420)
458 << "Non-I420 output buffer returned";
460 BrowserThread::PostTask(
461 BrowserThread::IO,
462 FROM_HERE,
463 base::Bind(
464 &VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread,
465 controller_,
466 buffer,
467 dimensions,
468 frame_rate,
469 timestamp));
472 void VideoCaptureController::VideoCaptureDeviceClient::OnError() {
473 BrowserThread::PostTask(BrowserThread::IO,
474 FROM_HERE,
475 base::Bind(&VideoCaptureController::DoErrorOnIOThread, controller_));
478 scoped_refptr<media::VideoCaptureDevice::Client::Buffer>
479 VideoCaptureController::VideoCaptureDeviceClient::DoReserveOutputBuffer(
480 media::VideoFrame::Format format,
481 const gfx::Size& dimensions,
482 int rotation) {
483 // The capture pipeline expects I420 for now.
484 DCHECK_EQ(format, media::VideoFrame::I420)
485 << "Non-I420 output buffer requested";
487 int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
488 const size_t frame_bytes =
489 media::VideoFrame::AllocationSize(format, dimensions);
491 int buffer_id =
492 buffer_pool_->ReserveForProducer(frame_bytes, &buffer_id_to_drop);
493 if (buffer_id == VideoCaptureBufferPool::kInvalidId)
494 return NULL;
495 void* data;
496 size_t size;
497 buffer_pool_->GetBufferInfo(buffer_id, &data, &size);
499 scoped_refptr<media::VideoCaptureDevice::Client::Buffer> output_buffer(
500 new PoolBuffer(buffer_pool_, buffer_id, data, size));
502 if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
503 BrowserThread::PostTask(BrowserThread::IO,
504 FROM_HERE,
505 base::Bind(&VideoCaptureController::DoBufferDestroyedOnIOThread,
506 controller_, buffer_id_to_drop));
507 rotated_buffers_.erase(buffer_id_to_drop);
510 // If a 90/270 rotation is required, letterboxing will be required. If the
511 // returned frame has not been rotated before, then the letterbox borders will
512 // not yet have been cleared and we should clear them now.
513 if ((rotation % 180) == 0) {
514 rotated_buffers_.erase(buffer_id);
515 } else {
516 if (rotated_buffers_.insert(buffer_id).second) {
517 scoped_refptr<media::VideoFrame> frame =
518 media::VideoFrame::WrapExternalPackedMemory(
519 media::VideoFrame::I420,
520 dimensions,
521 gfx::Rect(dimensions),
522 dimensions,
523 static_cast<uint8*>(output_buffer->data()),
524 output_buffer->size(),
525 base::SharedMemory::NULLHandle(),
526 base::TimeDelta(),
527 base::Closure());
528 media::FillYUV(frame, 0, 128, 128);
532 return output_buffer;
535 VideoCaptureController::~VideoCaptureController() {
536 STLDeleteContainerPointers(controller_clients_.begin(),
537 controller_clients_.end());
540 void VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread(
541 scoped_refptr<media::VideoCaptureDevice::Client::Buffer> buffer,
542 const gfx::Size& dimensions,
543 int frame_rate,
544 base::Time timestamp) {
545 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
546 DCHECK_NE(buffer->id(), VideoCaptureBufferPool::kInvalidId);
548 VideoCaptureFormat frame_format(
549 dimensions, frame_rate, media::PIXEL_FORMAT_I420);
551 int count = 0;
552 if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
553 for (ControllerClients::iterator client_it = controller_clients_.begin();
554 client_it != controller_clients_.end(); ++client_it) {
555 ControllerClient* client = *client_it;
556 if (client->session_closed)
557 continue;
559 bool is_new_buffer = client->known_buffers.insert(buffer->id()).second;
560 if (is_new_buffer) {
561 // On the first use of a buffer on a client, share the memory handle.
562 size_t memory_size = 0;
563 base::SharedMemoryHandle remote_handle = buffer_pool_->ShareToProcess(
564 buffer->id(), client->render_process_handle, &memory_size);
565 client->event_handler->OnBufferCreated(
566 client->controller_id, remote_handle, memory_size, buffer->id());
569 client->event_handler->OnBufferReady(
570 client->controller_id, buffer->id(), timestamp, frame_format);
571 bool inserted = client->active_buffers.insert(buffer->id()).second;
572 DCHECK(inserted) << "Unexpected duplicate buffer: " << buffer->id();
573 count++;
577 buffer_pool_->HoldForConsumers(buffer->id(), count);
580 void VideoCaptureController::DoErrorOnIOThread() {
581 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
582 state_ = VIDEO_CAPTURE_STATE_ERROR;
584 for (ControllerClients::iterator client_it = controller_clients_.begin();
585 client_it != controller_clients_.end(); ++client_it) {
586 ControllerClient* client = *client_it;
587 if (client->session_closed)
588 continue;
590 client->event_handler->OnError(client->controller_id);
594 void VideoCaptureController::DoBufferDestroyedOnIOThread(
595 int buffer_id_to_drop) {
596 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
598 for (ControllerClients::iterator client_it = controller_clients_.begin();
599 client_it != controller_clients_.end(); ++client_it) {
600 ControllerClient* client = *client_it;
601 if (client->session_closed)
602 continue;
604 if (client->known_buffers.erase(buffer_id_to_drop)) {
605 client->event_handler->OnBufferDestroyed(client->controller_id,
606 buffer_id_to_drop);
611 VideoCaptureController::ControllerClient*
612 VideoCaptureController::FindClient(
613 const VideoCaptureControllerID& id,
614 VideoCaptureControllerEventHandler* handler,
615 const ControllerClients& clients) {
616 for (ControllerClients::const_iterator client_it = clients.begin();
617 client_it != clients.end(); ++client_it) {
618 if ((*client_it)->controller_id == id &&
619 (*client_it)->event_handler == handler) {
620 return *client_it;
623 return NULL;
626 VideoCaptureController::ControllerClient*
627 VideoCaptureController::FindClient(
628 int session_id,
629 const ControllerClients& clients) {
630 for (ControllerClients::const_iterator client_it = clients.begin();
631 client_it != clients.end(); ++client_it) {
632 if ((*client_it)->session_id == session_id) {
633 return *client_it;
636 return NULL;
639 int VideoCaptureController::GetClientCount() {
640 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
641 return controller_clients_.size();
644 } // namespace content