Revert 268405 "Make sure that ScratchBuffer::Allocate() always r..."
[chromium-blink-merge.git] / content / browser / renderer_host / media / video_capture_controller.cc
blobdeb5e198ce7ede26305bc3fc904086f35b4b7649
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/renderer_host/media/video_capture_controller.h"
7 #include <map>
8 #include <set>
10 #include "base/bind.h"
11 #include "base/debug/trace_event.h"
12 #include "base/metrics/histogram.h"
13 #include "base/metrics/sparse_histogram.h"
14 #include "base/stl_util.h"
15 #include "content/browser/renderer_host/media/media_stream_manager.h"
16 #include "content/browser/renderer_host/media/video_capture_manager.h"
17 #include "content/public/browser/browser_thread.h"
18 #include "gpu/command_buffer/common/mailbox_holder.h"
19 #include "media/base/video_frame.h"
20 #include "media/base/video_util.h"
21 #include "media/base/yuv_convert.h"
22 #include "third_party/libyuv/include/libyuv.h"
24 using media::VideoCaptureFormat;
26 namespace content {
28 namespace {
30 static const int kInfiniteRatio = 99999;
32 #define UMA_HISTOGRAM_ASPECT_RATIO(name, width, height) \
33 UMA_HISTOGRAM_SPARSE_SLOWLY( \
34 name, \
35 (height) ? ((width) * 100) / (height) : kInfiniteRatio);
37 // The number of buffers that VideoCaptureBufferPool should allocate.
38 const int kNoOfBuffers = 3;
40 class PoolBuffer : public media::VideoCaptureDevice::Client::Buffer {
41 public:
42 PoolBuffer(const scoped_refptr<VideoCaptureBufferPool>& pool,
43 int buffer_id,
44 void* data,
45 size_t size)
46 : Buffer(buffer_id, data, size), pool_(pool) {
47 DCHECK(pool_);
50 private:
51 virtual ~PoolBuffer() { pool_->RelinquishProducerReservation(id()); }
53 const scoped_refptr<VideoCaptureBufferPool> pool_;
56 } // anonymous namespace
58 struct VideoCaptureController::ControllerClient {
59 ControllerClient(const VideoCaptureControllerID& id,
60 VideoCaptureControllerEventHandler* handler,
61 base::ProcessHandle render_process,
62 media::VideoCaptureSessionId session_id,
63 const media::VideoCaptureParams& params)
64 : controller_id(id),
65 event_handler(handler),
66 render_process_handle(render_process),
67 session_id(session_id),
68 parameters(params),
69 session_closed(false) {}
71 ~ControllerClient() {}
73 // ID used for identifying this object.
74 const VideoCaptureControllerID controller_id;
75 VideoCaptureControllerEventHandler* const event_handler;
77 // Handle to the render process that will receive the capture buffers.
78 const base::ProcessHandle render_process_handle;
79 const media::VideoCaptureSessionId session_id;
80 const media::VideoCaptureParams parameters;
82 // Buffers that are currently known to this client.
83 std::set<int> known_buffers;
85 // Buffers currently held by this client, and syncpoint callback to call when
86 // they are returned from the client.
87 typedef std::map<int, scoped_refptr<media::VideoFrame> > ActiveBufferMap;
88 ActiveBufferMap active_buffers;
90 // State of capture session, controlled by VideoCaptureManager directly. This
91 // transitions to true as soon as StopSession() occurs, at which point the
92 // client is sent an OnEnded() event. However, because the client retains a
93 // VideoCaptureController* pointer, its ControllerClient entry lives on until
94 // it unregisters itself via RemoveClient(), which may happen asynchronously.
96 // TODO(nick): If we changed the semantics of VideoCaptureHost so that
97 // OnEnded() events were processed synchronously (with the RemoveClient() done
98 // implicitly), we could avoid tracking this state here in the Controller, and
99 // simplify the code in both places.
100 bool session_closed;
103 // Receives events from the VideoCaptureDevice and posts them to a
104 // VideoCaptureController on the IO thread. An instance of this class may safely
105 // outlive its target VideoCaptureController.
107 // Methods of this class may be called from any thread, and in practice will
108 // often be called on some auxiliary thread depending on the platform and the
109 // device type; including, for example, the DirectShow thread on Windows, the
110 // v4l2_thread on Linux, and the UI thread for tab capture.
111 class VideoCaptureController::VideoCaptureDeviceClient
112 : public media::VideoCaptureDevice::Client {
113 public:
114 explicit VideoCaptureDeviceClient(
115 const base::WeakPtr<VideoCaptureController>& controller,
116 const scoped_refptr<VideoCaptureBufferPool>& buffer_pool);
117 virtual ~VideoCaptureDeviceClient();
119 // VideoCaptureDevice::Client implementation.
120 virtual scoped_refptr<Buffer> ReserveOutputBuffer(
121 media::VideoFrame::Format format,
122 const gfx::Size& size) OVERRIDE;
123 virtual void OnIncomingCapturedData(const uint8* data,
124 int length,
125 const VideoCaptureFormat& frame_format,
126 int rotation,
127 base::TimeTicks timestamp) OVERRIDE;
128 virtual void OnIncomingCapturedVideoFrame(
129 const scoped_refptr<Buffer>& buffer,
130 const VideoCaptureFormat& buffer_format,
131 const scoped_refptr<media::VideoFrame>& frame,
132 base::TimeTicks timestamp) OVERRIDE;
133 virtual void OnError(const std::string& reason) OVERRIDE;
135 private:
136 scoped_refptr<Buffer> DoReserveOutputBuffer(media::VideoFrame::Format format,
137 const gfx::Size& dimensions);
139 // The controller to which we post events.
140 const base::WeakPtr<VideoCaptureController> controller_;
142 // The pool of shared-memory buffers used for capturing.
143 const scoped_refptr<VideoCaptureBufferPool> buffer_pool_;
145 bool first_frame_;
148 VideoCaptureController::VideoCaptureController()
149 : buffer_pool_(new VideoCaptureBufferPool(kNoOfBuffers)),
150 state_(VIDEO_CAPTURE_STATE_STARTED),
151 weak_ptr_factory_(this) {
154 VideoCaptureController::VideoCaptureDeviceClient::VideoCaptureDeviceClient(
155 const base::WeakPtr<VideoCaptureController>& controller,
156 const scoped_refptr<VideoCaptureBufferPool>& buffer_pool)
157 : controller_(controller), buffer_pool_(buffer_pool), first_frame_(true) {}
159 VideoCaptureController::VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {}
161 base::WeakPtr<VideoCaptureController> VideoCaptureController::GetWeakPtr() {
162 return weak_ptr_factory_.GetWeakPtr();
165 scoped_ptr<media::VideoCaptureDevice::Client>
166 VideoCaptureController::NewDeviceClient() {
167 scoped_ptr<media::VideoCaptureDevice::Client> result(
168 new VideoCaptureDeviceClient(this->GetWeakPtr(), buffer_pool_));
169 return result.Pass();
172 void VideoCaptureController::AddClient(
173 const VideoCaptureControllerID& id,
174 VideoCaptureControllerEventHandler* event_handler,
175 base::ProcessHandle render_process,
176 media::VideoCaptureSessionId session_id,
177 const media::VideoCaptureParams& params) {
178 DCHECK_CURRENTLY_ON(BrowserThread::IO);
179 DVLOG(1) << "VideoCaptureController::AddClient, id " << id.device_id
180 << ", " << params.requested_format.frame_size.ToString()
181 << ", " << params.requested_format.frame_rate
182 << ", " << session_id
183 << ")";
185 // If this is the first client added to the controller, cache the parameters.
186 if (!controller_clients_.size())
187 video_capture_format_ = params.requested_format;
189 // Signal error in case device is already in error state.
190 if (state_ == VIDEO_CAPTURE_STATE_ERROR) {
191 event_handler->OnError(id);
192 return;
195 // Do nothing if this client has called AddClient before.
196 if (FindClient(id, event_handler, controller_clients_))
197 return;
199 ControllerClient* client = new ControllerClient(
200 id, event_handler, render_process, session_id, params);
201 // If we already have gotten frame_info from the device, repeat it to the new
202 // client.
203 if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
204 controller_clients_.push_back(client);
205 return;
209 int VideoCaptureController::RemoveClient(
210 const VideoCaptureControllerID& id,
211 VideoCaptureControllerEventHandler* event_handler) {
212 DCHECK_CURRENTLY_ON(BrowserThread::IO);
213 DVLOG(1) << "VideoCaptureController::RemoveClient, id " << id.device_id;
215 ControllerClient* client = FindClient(id, event_handler, controller_clients_);
216 if (!client)
217 return kInvalidMediaCaptureSessionId;
219 // Take back all buffers held by the |client|.
220 for (ControllerClient::ActiveBufferMap::iterator buffer_it =
221 client->active_buffers.begin();
222 buffer_it != client->active_buffers.end();
223 ++buffer_it) {
224 buffer_pool_->RelinquishConsumerHold(buffer_it->first, 1);
226 client->active_buffers.clear();
228 int session_id = client->session_id;
229 controller_clients_.remove(client);
230 delete client;
232 return session_id;
235 void VideoCaptureController::StopSession(int session_id) {
236 DCHECK_CURRENTLY_ON(BrowserThread::IO);
237 DVLOG(1) << "VideoCaptureController::StopSession, id " << session_id;
239 ControllerClient* client = FindClient(session_id, controller_clients_);
241 if (client) {
242 client->session_closed = true;
243 client->event_handler->OnEnded(client->controller_id);
247 void VideoCaptureController::ReturnBuffer(
248 const VideoCaptureControllerID& id,
249 VideoCaptureControllerEventHandler* event_handler,
250 int buffer_id,
251 const std::vector<uint32>& sync_points) {
252 DCHECK_CURRENTLY_ON(BrowserThread::IO);
254 ControllerClient* client = FindClient(id, event_handler, controller_clients_);
256 // If this buffer is not held by this client, or this client doesn't exist
257 // in controller, do nothing.
258 ControllerClient::ActiveBufferMap::iterator iter;
259 if (!client || (iter = client->active_buffers.find(buffer_id)) ==
260 client->active_buffers.end()) {
261 NOTREACHED();
262 return;
264 scoped_refptr<media::VideoFrame> frame = iter->second;
265 client->active_buffers.erase(iter);
267 if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
268 for (size_t i = 0; i < sync_points.size(); i++)
269 frame->AppendReleaseSyncPoint(sync_points[i]);
272 buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
275 const media::VideoCaptureFormat&
276 VideoCaptureController::GetVideoCaptureFormat() const {
277 DCHECK_CURRENTLY_ON(BrowserThread::IO);
278 return video_capture_format_;
281 scoped_refptr<media::VideoCaptureDevice::Client::Buffer>
282 VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer(
283 media::VideoFrame::Format format,
284 const gfx::Size& size) {
285 return DoReserveOutputBuffer(format, size);
288 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedData(
289 const uint8* data,
290 int length,
291 const VideoCaptureFormat& frame_format,
292 int rotation,
293 base::TimeTicks timestamp) {
294 TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedData");
296 if (!frame_format.IsValid())
297 return;
299 // Chopped pixels in width/height in case video capture device has odd
300 // numbers for width/height.
301 int chopped_width = 0;
302 int chopped_height = 0;
303 int new_unrotated_width = frame_format.frame_size.width();
304 int new_unrotated_height = frame_format.frame_size.height();
306 if (new_unrotated_width & 1) {
307 --new_unrotated_width;
308 chopped_width = 1;
310 if (new_unrotated_height & 1) {
311 --new_unrotated_height;
312 chopped_height = 1;
315 int destination_width = new_unrotated_width;
316 int destination_height = new_unrotated_height;
317 if (rotation == 90 || rotation == 270) {
318 destination_width = new_unrotated_height;
319 destination_height = new_unrotated_width;
321 const gfx::Size dimensions(destination_width, destination_height);
322 if (!media::VideoFrame::IsValidConfig(media::VideoFrame::I420,
323 dimensions,
324 gfx::Rect(dimensions),
325 dimensions)) {
326 return;
329 scoped_refptr<Buffer> buffer =
330 DoReserveOutputBuffer(media::VideoFrame::I420, dimensions);
332 if (!buffer)
333 return;
334 uint8* yplane = NULL;
335 bool flip = false;
336 yplane = reinterpret_cast<uint8*>(buffer->data());
337 uint8* uplane =
338 yplane +
339 media::VideoFrame::PlaneAllocationSize(
340 media::VideoFrame::I420, media::VideoFrame::kYPlane, dimensions);
341 uint8* vplane =
342 uplane +
343 media::VideoFrame::PlaneAllocationSize(
344 media::VideoFrame::I420, media::VideoFrame::kUPlane, dimensions);
345 int yplane_stride = dimensions.width();
346 int uv_plane_stride = yplane_stride / 2;
347 int crop_x = 0;
348 int crop_y = 0;
349 libyuv::FourCC origin_colorspace = libyuv::FOURCC_ANY;
351 libyuv::RotationMode rotation_mode = libyuv::kRotate0;
352 if (rotation == 90)
353 rotation_mode = libyuv::kRotate90;
354 else if (rotation == 180)
355 rotation_mode = libyuv::kRotate180;
356 else if (rotation == 270)
357 rotation_mode = libyuv::kRotate270;
359 switch (frame_format.pixel_format) {
360 case media::PIXEL_FORMAT_UNKNOWN: // Color format not set.
361 break;
362 case media::PIXEL_FORMAT_I420:
363 DCHECK(!chopped_width && !chopped_height);
364 origin_colorspace = libyuv::FOURCC_I420;
365 break;
366 case media::PIXEL_FORMAT_YV12:
367 DCHECK(!chopped_width && !chopped_height);
368 origin_colorspace = libyuv::FOURCC_YV12;
369 break;
370 case media::PIXEL_FORMAT_NV21:
371 DCHECK(!chopped_width && !chopped_height);
372 origin_colorspace = libyuv::FOURCC_NV21;
373 break;
374 case media::PIXEL_FORMAT_YUY2:
375 DCHECK(!chopped_width && !chopped_height);
376 origin_colorspace = libyuv::FOURCC_YUY2;
377 break;
378 case media::PIXEL_FORMAT_UYVY:
379 DCHECK(!chopped_width && !chopped_height);
380 origin_colorspace = libyuv::FOURCC_UYVY;
381 break;
382 case media::PIXEL_FORMAT_RGB24:
383 origin_colorspace = libyuv::FOURCC_24BG;
384 #if defined(OS_WIN)
385 // TODO(wjia): Currently, for RGB24 on WIN, capture device always
386 // passes in positive src_width and src_height. Remove this hardcoded
387 // value when nagative src_height is supported. The negative src_height
388 // indicates that vertical flipping is needed.
389 flip = true;
390 #endif
391 break;
392 case media::PIXEL_FORMAT_ARGB:
393 origin_colorspace = libyuv::FOURCC_ARGB;
394 break;
395 case media::PIXEL_FORMAT_MJPEG:
396 origin_colorspace = libyuv::FOURCC_MJPG;
397 break;
398 default:
399 NOTREACHED();
402 libyuv::ConvertToI420(data,
403 length,
404 yplane,
405 yplane_stride,
406 uplane,
407 uv_plane_stride,
408 vplane,
409 uv_plane_stride,
410 crop_x,
411 crop_y,
412 frame_format.frame_size.width(),
413 (flip ? -frame_format.frame_size.height() :
414 frame_format.frame_size.height()),
415 new_unrotated_width,
416 new_unrotated_height,
417 rotation_mode,
418 origin_colorspace);
419 scoped_refptr<media::VideoFrame> frame =
420 media::VideoFrame::WrapExternalPackedMemory(
421 media::VideoFrame::I420,
422 dimensions,
423 gfx::Rect(dimensions),
424 dimensions,
425 yplane,
426 media::VideoFrame::AllocationSize(media::VideoFrame::I420,
427 dimensions),
428 base::SharedMemory::NULLHandle(),
429 base::TimeDelta(),
430 base::Closure());
431 DCHECK(frame);
433 VideoCaptureFormat format(
434 dimensions, frame_format.frame_rate, media::PIXEL_FORMAT_I420);
435 BrowserThread::PostTask(
436 BrowserThread::IO,
437 FROM_HERE,
438 base::Bind(
439 &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
440 controller_,
441 buffer,
442 format,
443 frame,
444 timestamp));
446 if (first_frame_) {
447 UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Width",
448 frame_format.frame_size.width());
449 UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Height",
450 frame_format.frame_size.height());
451 UMA_HISTOGRAM_ASPECT_RATIO("Media.VideoCapture.AspectRatio",
452 frame_format.frame_size.width(),
453 frame_format.frame_size.height());
454 UMA_HISTOGRAM_COUNTS("Media.VideoCapture.FrameRate",
455 frame_format.frame_rate);
456 UMA_HISTOGRAM_ENUMERATION("Media.VideoCapture.PixelFormat",
457 frame_format.pixel_format,
458 media::PIXEL_FORMAT_MAX);
459 first_frame_ = false;
463 void
464 VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedVideoFrame(
465 const scoped_refptr<Buffer>& buffer,
466 const VideoCaptureFormat& buffer_format,
467 const scoped_refptr<media::VideoFrame>& frame,
468 base::TimeTicks timestamp) {
469 BrowserThread::PostTask(
470 BrowserThread::IO,
471 FROM_HERE,
472 base::Bind(
473 &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
474 controller_,
475 buffer,
476 buffer_format,
477 frame,
478 timestamp));
481 void VideoCaptureController::VideoCaptureDeviceClient::OnError(
482 const std::string& reason) {
483 MediaStreamManager::SendMessageToNativeLog(
484 "Error on video capture: " + reason);
485 BrowserThread::PostTask(BrowserThread::IO,
486 FROM_HERE,
487 base::Bind(&VideoCaptureController::DoErrorOnIOThread, controller_));
490 scoped_refptr<media::VideoCaptureDevice::Client::Buffer>
491 VideoCaptureController::VideoCaptureDeviceClient::DoReserveOutputBuffer(
492 media::VideoFrame::Format format,
493 const gfx::Size& dimensions) {
494 size_t frame_bytes = 0;
495 if (format == media::VideoFrame::NATIVE_TEXTURE) {
496 DCHECK_EQ(dimensions.width(), 0);
497 DCHECK_EQ(dimensions.height(), 0);
498 } else {
499 // The capture pipeline expects I420 for now.
500 DCHECK_EQ(format, media::VideoFrame::I420)
501 << "Non-I420 output buffer format " << format << " requested";
502 frame_bytes = media::VideoFrame::AllocationSize(format, dimensions);
505 int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
506 int buffer_id =
507 buffer_pool_->ReserveForProducer(frame_bytes, &buffer_id_to_drop);
508 if (buffer_id == VideoCaptureBufferPool::kInvalidId)
509 return NULL;
510 void* data;
511 size_t size;
512 buffer_pool_->GetBufferInfo(buffer_id, &data, &size);
514 scoped_refptr<media::VideoCaptureDevice::Client::Buffer> output_buffer(
515 new PoolBuffer(buffer_pool_, buffer_id, data, size));
517 if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
518 BrowserThread::PostTask(BrowserThread::IO,
519 FROM_HERE,
520 base::Bind(&VideoCaptureController::DoBufferDestroyedOnIOThread,
521 controller_, buffer_id_to_drop));
524 return output_buffer;
527 VideoCaptureController::~VideoCaptureController() {
528 STLDeleteContainerPointers(controller_clients_.begin(),
529 controller_clients_.end());
532 void VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread(
533 const scoped_refptr<media::VideoCaptureDevice::Client::Buffer>& buffer,
534 const media::VideoCaptureFormat& buffer_format,
535 const scoped_refptr<media::VideoFrame>& frame,
536 base::TimeTicks timestamp) {
537 DCHECK_CURRENTLY_ON(BrowserThread::IO);
538 DCHECK_NE(buffer->id(), VideoCaptureBufferPool::kInvalidId);
540 int count = 0;
541 if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
542 for (ControllerClients::iterator client_it = controller_clients_.begin();
543 client_it != controller_clients_.end(); ++client_it) {
544 ControllerClient* client = *client_it;
545 if (client->session_closed)
546 continue;
548 if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
549 client->event_handler->OnMailboxBufferReady(client->controller_id,
550 buffer->id(),
551 *frame->mailbox_holder(),
552 buffer_format,
553 timestamp);
554 } else {
555 bool is_new_buffer = client->known_buffers.insert(buffer->id()).second;
556 if (is_new_buffer) {
557 // On the first use of a buffer on a client, share the memory handle.
558 size_t memory_size = 0;
559 base::SharedMemoryHandle remote_handle = buffer_pool_->ShareToProcess(
560 buffer->id(), client->render_process_handle, &memory_size);
561 client->event_handler->OnBufferCreated(
562 client->controller_id, remote_handle, memory_size, buffer->id());
565 client->event_handler->OnBufferReady(
566 client->controller_id, buffer->id(), buffer_format, timestamp);
569 bool inserted =
570 client->active_buffers.insert(std::make_pair(buffer->id(), frame))
571 .second;
572 DCHECK(inserted) << "Unexpected duplicate buffer: " << buffer->id();
573 count++;
577 buffer_pool_->HoldForConsumers(buffer->id(), count);
580 void VideoCaptureController::DoErrorOnIOThread() {
581 DCHECK_CURRENTLY_ON(BrowserThread::IO);
582 state_ = VIDEO_CAPTURE_STATE_ERROR;
584 for (ControllerClients::iterator client_it = controller_clients_.begin();
585 client_it != controller_clients_.end(); ++client_it) {
586 ControllerClient* client = *client_it;
587 if (client->session_closed)
588 continue;
590 client->event_handler->OnError(client->controller_id);
594 void VideoCaptureController::DoBufferDestroyedOnIOThread(
595 int buffer_id_to_drop) {
596 DCHECK_CURRENTLY_ON(BrowserThread::IO);
598 for (ControllerClients::iterator client_it = controller_clients_.begin();
599 client_it != controller_clients_.end(); ++client_it) {
600 ControllerClient* client = *client_it;
601 if (client->session_closed)
602 continue;
604 if (client->known_buffers.erase(buffer_id_to_drop)) {
605 client->event_handler->OnBufferDestroyed(client->controller_id,
606 buffer_id_to_drop);
611 VideoCaptureController::ControllerClient*
612 VideoCaptureController::FindClient(
613 const VideoCaptureControllerID& id,
614 VideoCaptureControllerEventHandler* handler,
615 const ControllerClients& clients) {
616 for (ControllerClients::const_iterator client_it = clients.begin();
617 client_it != clients.end(); ++client_it) {
618 if ((*client_it)->controller_id == id &&
619 (*client_it)->event_handler == handler) {
620 return *client_it;
623 return NULL;
626 VideoCaptureController::ControllerClient*
627 VideoCaptureController::FindClient(
628 int session_id,
629 const ControllerClients& clients) {
630 for (ControllerClients::const_iterator client_it = clients.begin();
631 client_it != clients.end(); ++client_it) {
632 if ((*client_it)->session_id == session_id) {
633 return *client_it;
636 return NULL;
639 int VideoCaptureController::GetClientCount() {
640 DCHECK_CURRENTLY_ON(BrowserThread::IO);
641 return controller_clients_.size();
644 } // namespace content