1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/renderer_host/media/video_capture_manager.h"
10 #include "base/bind_helpers.h"
11 #include "base/logging.h"
12 #include "base/message_loop/message_loop.h"
13 #include "base/stl_util.h"
14 #include "base/task_runner_util.h"
15 #include "base/threading/sequenced_worker_pool.h"
16 #include "content/browser/media/capture/web_contents_video_capture_device.h"
17 #include "content/browser/renderer_host/media/video_capture_controller.h"
18 #include "content/browser/renderer_host/media/video_capture_controller_event_handler.h"
19 #include "content/public/browser/browser_thread.h"
20 #include "content/public/browser/desktop_media_id.h"
21 #include "content/public/common/content_switches.h"
22 #include "content/public/common/media_stream_request.h"
23 #include "media/base/bind_to_current_loop.h"
24 #include "media/base/scoped_histogram_timer.h"
25 #include "media/video/capture/video_capture_device.h"
26 #include "media/video/capture/video_capture_device_factory.h"
28 #if defined(ENABLE_SCREEN_CAPTURE)
29 #include "content/browser/media/capture/desktop_capture_device.h"
31 #include "content/browser/media/capture/desktop_capture_device_aura.h"
37 // Compares two VideoCaptureFormat by checking smallest frame_size area, then
38 // by _largest_ frame_rate. Used to order a VideoCaptureFormats vector so that
39 // the first entry for a given resolution has the largest frame rate, as needed
40 // by the ConsolidateCaptureFormats() method.
41 bool IsCaptureFormatSmaller(const media::VideoCaptureFormat
& format1
,
42 const media::VideoCaptureFormat
& format2
) {
43 if (format1
.frame_size
.GetArea() == format2
.frame_size
.GetArea())
44 return format1
.frame_rate
> format2
.frame_rate
;
45 return format1
.frame_size
.GetArea() < format2
.frame_size
.GetArea();
48 bool IsCaptureFormatSizeEqual(const media::VideoCaptureFormat
& format1
,
49 const media::VideoCaptureFormat
& format2
) {
50 return format1
.frame_size
.GetArea() == format2
.frame_size
.GetArea();
53 // This function receives a list of capture formats, removes duplicated
54 // resolutions while keeping the highest frame rate for each, and forcing I420
56 void ConsolidateCaptureFormats(media::VideoCaptureFormats
* formats
) {
59 std::sort(formats
->begin(), formats
->end(), IsCaptureFormatSmaller
);
60 // Due to the ordering imposed, the largest frame_rate is kept while removing
61 // duplicated resolutions.
62 media::VideoCaptureFormats::iterator last
=
63 std::unique(formats
->begin(), formats
->end(), IsCaptureFormatSizeEqual
);
64 formats
->erase(last
, formats
->end());
65 // Mark all formats as I420, since this is what the renderer side will get
66 // anyhow: the actual pixel format is decided at the device level.
67 for (media::VideoCaptureFormats::iterator it
= formats
->begin();
68 it
!= formats
->end(); ++it
) {
69 it
->pixel_format
= media::PIXEL_FORMAT_I420
;
77 VideoCaptureManager::DeviceEntry::DeviceEntry(
78 MediaStreamType stream_type
,
79 const std::string
& id
,
80 scoped_ptr
<VideoCaptureController
> controller
)
81 : stream_type(stream_type
),
83 video_capture_controller(controller
.Pass()) {}
85 VideoCaptureManager::DeviceEntry::~DeviceEntry() {}
87 VideoCaptureManager::DeviceInfo::DeviceInfo() {}
89 VideoCaptureManager::DeviceInfo::DeviceInfo(
90 const media::VideoCaptureDevice::Name
& name
,
91 const media::VideoCaptureFormats
& supported_formats
)
93 supported_formats(supported_formats
) {}
95 VideoCaptureManager::DeviceInfo::~DeviceInfo() {}
97 VideoCaptureManager::VideoCaptureManager(
98 scoped_ptr
<media::VideoCaptureDeviceFactory
> factory
)
100 new_capture_session_id_(1),
101 video_capture_device_factory_(factory
.Pass()) {
104 VideoCaptureManager::~VideoCaptureManager() {
105 DCHECK(devices_
.empty());
108 void VideoCaptureManager::Register(
109 MediaStreamProviderListener
* listener
,
110 const scoped_refptr
<base::SingleThreadTaskRunner
>& device_task_runner
) {
111 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
113 DCHECK(!device_task_runner_
.get());
114 listener_
= listener
;
115 device_task_runner_
= device_task_runner
;
118 void VideoCaptureManager::Unregister() {
123 void VideoCaptureManager::EnumerateDevices(MediaStreamType stream_type
) {
124 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
125 DVLOG(1) << "VideoCaptureManager::EnumerateDevices, type " << stream_type
;
127 DCHECK_EQ(stream_type
, MEDIA_DEVICE_VIDEO_CAPTURE
);
129 // Bind a callback to ConsolidateDevicesInfoOnDeviceThread() with an argument
130 // for another callback to OnDevicesInfoEnumerated() to be run in the current
131 // loop, i.e. IO loop. Pass a timer for UMA histogram collection.
132 base::Callback
<void(scoped_ptr
<media::VideoCaptureDevice::Names
>)>
133 devices_enumerated_callback
=
134 base::Bind(&VideoCaptureManager::ConsolidateDevicesInfoOnDeviceThread
,
136 media::BindToCurrentLoop(base::Bind(
137 &VideoCaptureManager::OnDevicesInfoEnumerated
,
140 base::Owned(new base::ElapsedTimer()))),
142 devices_info_cache_
);
143 // OK to use base::Unretained() since we own the VCDFactory and |this| is
144 // bound in |devices_enumerated_callback|.
145 device_task_runner_
->PostTask(FROM_HERE
,
146 base::Bind(&media::VideoCaptureDeviceFactory::EnumerateDeviceNames
,
147 base::Unretained(video_capture_device_factory_
.get()),
148 devices_enumerated_callback
));
151 int VideoCaptureManager::Open(const StreamDeviceInfo
& device_info
) {
152 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
155 // Generate a new id for the session being opened.
156 const media::VideoCaptureSessionId capture_session_id
=
157 new_capture_session_id_
++;
159 DCHECK(sessions_
.find(capture_session_id
) == sessions_
.end());
160 DVLOG(1) << "VideoCaptureManager::Open, id " << capture_session_id
;
162 // We just save the stream info for processing later.
163 sessions_
[capture_session_id
] = device_info
.device
;
165 // Notify our listener asynchronously; this ensures that we return
166 // |capture_session_id| to the caller of this function before using that same
167 // id in a listener event.
168 base::MessageLoop::current()->PostTask(FROM_HERE
,
169 base::Bind(&VideoCaptureManager::OnOpened
, this,
170 device_info
.device
.type
, capture_session_id
));
171 return capture_session_id
;
174 void VideoCaptureManager::Close(int capture_session_id
) {
175 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
177 DVLOG(1) << "VideoCaptureManager::Close, id " << capture_session_id
;
179 SessionMap::iterator session_it
= sessions_
.find(capture_session_id
);
180 if (session_it
== sessions_
.end()) {
185 DeviceEntry
* const existing_device
= GetDeviceEntryForMediaStreamDevice(
187 if (existing_device
) {
188 // Remove any client that is still using the session. This is safe to call
189 // even if there are no clients using the session.
190 existing_device
->video_capture_controller
->StopSession(capture_session_id
);
192 // StopSession() may have removed the last client, so we might need to
194 DestroyDeviceEntryIfNoClients(existing_device
);
197 // Notify listeners asynchronously, and forget the session.
198 base::MessageLoop::current()->PostTask(FROM_HERE
,
199 base::Bind(&VideoCaptureManager::OnClosed
, this, session_it
->second
.type
,
200 capture_session_id
));
201 sessions_
.erase(session_it
);
204 void VideoCaptureManager::DoStartDeviceOnDeviceThread(
205 media::VideoCaptureSessionId session_id
,
207 const media::VideoCaptureParams
& params
,
208 scoped_ptr
<media::VideoCaptureDevice::Client
> device_client
) {
209 SCOPED_UMA_HISTOGRAM_TIMER("Media.VideoCaptureManager.StartDeviceTime");
210 DCHECK(IsOnDeviceThread());
212 scoped_ptr
<media::VideoCaptureDevice
> video_capture_device
;
213 switch (entry
->stream_type
) {
214 case MEDIA_DEVICE_VIDEO_CAPTURE
: {
215 // We look up the device id from the renderer in our local enumeration
216 // since the renderer does not have all the information that might be
217 // held in the browser-side VideoCaptureDevice::Name structure.
218 DeviceInfo
* found
= FindDeviceInfoById(entry
->id
, devices_info_cache_
);
220 video_capture_device
=
221 video_capture_device_factory_
->Create(found
->name
);
225 case MEDIA_TAB_VIDEO_CAPTURE
: {
226 video_capture_device
.reset(
227 WebContentsVideoCaptureDevice::Create(entry
->id
));
230 case MEDIA_DESKTOP_VIDEO_CAPTURE
: {
231 #if defined(ENABLE_SCREEN_CAPTURE)
232 DesktopMediaID id
= DesktopMediaID::Parse(entry
->id
);
233 #if defined(USE_AURA)
234 if (id
.type
== DesktopMediaID::TYPE_AURA_WINDOW
) {
235 video_capture_device
.reset(DesktopCaptureDeviceAura::Create(id
));
238 if (id
.type
!= DesktopMediaID::TYPE_NONE
&&
239 id
.type
!= DesktopMediaID::TYPE_AURA_WINDOW
) {
240 video_capture_device
= DesktopCaptureDevice::Create(id
);
241 if (notification_window_ids_
.find(session_id
) !=
242 notification_window_ids_
.end()) {
243 static_cast<DesktopCaptureDevice
*>(video_capture_device
.get())
244 ->SetNotificationWindowId(notification_window_ids_
[session_id
]);
247 #endif // defined(ENABLE_SCREEN_CAPTURE)
256 if (!video_capture_device
) {
257 device_client
->OnError("Could not create capture device");
261 video_capture_device
->AllocateAndStart(params
, device_client
.Pass());
262 entry
->video_capture_device
= video_capture_device
.Pass();
265 void VideoCaptureManager::StartCaptureForClient(
266 media::VideoCaptureSessionId session_id
,
267 const media::VideoCaptureParams
& params
,
268 base::ProcessHandle client_render_process
,
269 VideoCaptureControllerID client_id
,
270 VideoCaptureControllerEventHandler
* client_handler
,
271 const DoneCB
& done_cb
) {
272 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
273 DVLOG(1) << "VideoCaptureManager::StartCaptureForClient, "
274 << params
.requested_format
.frame_size
.ToString() << ", "
275 << params
.requested_format
.frame_rate
<< ", #" << session_id
<< ")";
277 DeviceEntry
* entry
= GetOrCreateDeviceEntry(session_id
);
279 done_cb
.Run(base::WeakPtr
<VideoCaptureController
>());
283 DCHECK(entry
->video_capture_controller
);
285 // First client starts the device.
286 if (entry
->video_capture_controller
->GetClientCount() == 0) {
287 DVLOG(1) << "VideoCaptureManager starting device (type = "
288 << entry
->stream_type
<< ", id = " << entry
->id
<< ")";
290 device_task_runner_
->PostTask(
293 &VideoCaptureManager::DoStartDeviceOnDeviceThread
,
298 base::Passed(entry
->video_capture_controller
->NewDeviceClient())));
300 // Run the callback first, as AddClient() may trigger OnFrameInfo().
301 done_cb
.Run(entry
->video_capture_controller
->GetWeakPtr());
302 entry
->video_capture_controller
->AddClient(
303 client_id
, client_handler
, client_render_process
, session_id
, params
);
306 void VideoCaptureManager::StopCaptureForClient(
307 VideoCaptureController
* controller
,
308 VideoCaptureControllerID client_id
,
309 VideoCaptureControllerEventHandler
* client_handler
,
310 bool aborted_due_to_error
) {
311 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
313 DCHECK(client_handler
);
315 DeviceEntry
* entry
= GetDeviceEntryForController(controller
);
320 if (aborted_due_to_error
) {
321 SessionMap::iterator it
;
322 for (it
= sessions_
.begin(); it
!= sessions_
.end(); ++it
) {
323 if (it
->second
.type
== entry
->stream_type
&&
324 it
->second
.id
== entry
->id
) {
325 listener_
->Aborted(it
->second
.type
, it
->first
);
331 // Detach client from controller.
332 media::VideoCaptureSessionId session_id
=
333 controller
->RemoveClient(client_id
, client_handler
);
334 DVLOG(1) << "VideoCaptureManager::StopCaptureForClient, session_id = "
337 // If controller has no more clients, delete controller and device.
338 DestroyDeviceEntryIfNoClients(entry
);
341 bool VideoCaptureManager::GetDeviceSupportedFormats(
342 media::VideoCaptureSessionId capture_session_id
,
343 media::VideoCaptureFormats
* supported_formats
) {
344 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
345 DCHECK(supported_formats
->empty());
347 SessionMap::iterator it
= sessions_
.find(capture_session_id
);
348 if (it
== sessions_
.end())
350 DVLOG(1) << "GetDeviceSupportedFormats for device: " << it
->second
.name
;
352 // Return all available formats of the device, regardless its started state.
353 DeviceInfo
* existing_device
=
354 FindDeviceInfoById(it
->second
.id
, devices_info_cache_
);
356 *supported_formats
= existing_device
->supported_formats
;
360 bool VideoCaptureManager::GetDeviceFormatsInUse(
361 media::VideoCaptureSessionId capture_session_id
,
362 media::VideoCaptureFormats
* formats_in_use
) {
363 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
364 DCHECK(formats_in_use
->empty());
366 SessionMap::iterator it
= sessions_
.find(capture_session_id
);
367 if (it
== sessions_
.end())
369 DVLOG(1) << "GetDeviceFormatsInUse for device: " << it
->second
.name
;
371 // Return the currently in-use format(s) of the device, if it's started.
372 DeviceEntry
* device_in_use
=
373 GetDeviceEntryForMediaStreamDevice(it
->second
);
375 // Currently only one format-in-use is supported at the VCC level.
376 formats_in_use
->push_back(
377 device_in_use
->video_capture_controller
->GetVideoCaptureFormat());
382 void VideoCaptureManager::SetDesktopCaptureWindowId(
383 media::VideoCaptureSessionId session_id
,
384 gfx::NativeViewId window_id
) {
385 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
386 SessionMap::iterator session_it
= sessions_
.find(session_id
);
387 if (session_it
== sessions_
.end()) {
388 device_task_runner_
->PostTask(
391 &VideoCaptureManager::SaveDesktopCaptureWindowIdOnDeviceThread
,
398 DeviceEntry
* const existing_device
=
399 GetDeviceEntryForMediaStreamDevice(session_it
->second
);
400 if (!existing_device
)
403 DCHECK_EQ(MEDIA_DESKTOP_VIDEO_CAPTURE
, existing_device
->stream_type
);
404 DesktopMediaID id
= DesktopMediaID::Parse(existing_device
->id
);
405 if (id
.type
== DesktopMediaID::TYPE_NONE
||
406 id
.type
== DesktopMediaID::TYPE_AURA_WINDOW
) {
410 device_task_runner_
->PostTask(
412 base::Bind(&VideoCaptureManager::SetDesktopCaptureWindowIdOnDeviceThread
,
418 void VideoCaptureManager::DoStopDeviceOnDeviceThread(DeviceEntry
* entry
) {
419 SCOPED_UMA_HISTOGRAM_TIMER("Media.VideoCaptureManager.StopDeviceTime");
420 DCHECK(IsOnDeviceThread());
421 if (entry
->video_capture_device
) {
422 entry
->video_capture_device
->StopAndDeAllocate();
424 entry
->video_capture_device
.reset();
427 void VideoCaptureManager::OnOpened(
428 MediaStreamType stream_type
,
429 media::VideoCaptureSessionId capture_session_id
) {
430 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
432 // Listener has been removed.
435 listener_
->Opened(stream_type
, capture_session_id
);
438 void VideoCaptureManager::OnClosed(
439 MediaStreamType stream_type
,
440 media::VideoCaptureSessionId capture_session_id
) {
441 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
443 // Listener has been removed.
446 listener_
->Closed(stream_type
, capture_session_id
);
449 void VideoCaptureManager::OnDevicesInfoEnumerated(
450 MediaStreamType stream_type
,
451 base::ElapsedTimer
* timer
,
452 const DeviceInfos
& new_devices_info_cache
) {
453 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
455 "Media.VideoCaptureManager.GetAvailableDevicesInfoOnDeviceThreadTime",
458 // Listener has been removed.
461 devices_info_cache_
= new_devices_info_cache
;
463 // Walk the |devices_info_cache_| and transform from VCD::Name to
464 // StreamDeviceInfo for return purposes.
465 StreamDeviceInfoArray devices
;
466 for (DeviceInfos::const_iterator it
= devices_info_cache_
.begin();
467 it
!= devices_info_cache_
.end(); ++it
) {
468 devices
.push_back(StreamDeviceInfo(
469 stream_type
, it
->name
.GetNameAndModel(), it
->name
.id()));
471 listener_
->DevicesEnumerated(stream_type
, devices
);
474 bool VideoCaptureManager::IsOnDeviceThread() const {
475 return device_task_runner_
->BelongsToCurrentThread();
478 void VideoCaptureManager::ConsolidateDevicesInfoOnDeviceThread(
479 base::Callback
<void(const DeviceInfos
&)> on_devices_enumerated_callback
,
480 MediaStreamType stream_type
,
481 const DeviceInfos
& old_device_info_cache
,
482 scoped_ptr
<media::VideoCaptureDevice::Names
> names_snapshot
) {
483 DCHECK(IsOnDeviceThread());
484 // Construct |new_devices_info_cache| with the cached devices that are still
485 // present in the system, and remove their names from |names_snapshot|, so we
486 // keep there the truly new devices.
487 DeviceInfos new_devices_info_cache
;
488 for (DeviceInfos::const_iterator it_device_info
=
489 old_device_info_cache
.begin();
490 it_device_info
!= old_device_info_cache
.end(); ++it_device_info
) {
491 for (media::VideoCaptureDevice::Names::iterator it
=
492 names_snapshot
->begin();
493 it
!= names_snapshot
->end(); ++it
) {
494 if (it_device_info
->name
.id() == it
->id()) {
495 new_devices_info_cache
.push_back(*it_device_info
);
496 names_snapshot
->erase(it
);
502 // Get the supported capture formats for the new devices in |names_snapshot|.
503 for (media::VideoCaptureDevice::Names::const_iterator it
=
504 names_snapshot
->begin();
505 it
!= names_snapshot
->end(); ++it
) {
506 media::VideoCaptureFormats supported_formats
;
507 DeviceInfo
device_info(*it
, media::VideoCaptureFormats());
508 video_capture_device_factory_
->GetDeviceSupportedFormats(
509 *it
, &(device_info
.supported_formats
));
510 ConsolidateCaptureFormats(&device_info
.supported_formats
);
511 new_devices_info_cache
.push_back(device_info
);
514 on_devices_enumerated_callback
.Run(new_devices_info_cache
);
517 VideoCaptureManager::DeviceEntry
*
518 VideoCaptureManager::GetDeviceEntryForMediaStreamDevice(
519 const MediaStreamDevice
& device_info
) {
520 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
522 for (DeviceEntries::iterator it
= devices_
.begin();
523 it
!= devices_
.end(); ++it
) {
524 DeviceEntry
* device
= *it
;
525 if (device_info
.type
== device
->stream_type
&&
526 device_info
.id
== device
->id
) {
533 VideoCaptureManager::DeviceEntry
*
534 VideoCaptureManager::GetDeviceEntryForController(
535 const VideoCaptureController
* controller
) const {
536 // Look up |controller| in |devices_|.
537 for (DeviceEntries::const_iterator it
= devices_
.begin();
538 it
!= devices_
.end(); ++it
) {
539 if ((*it
)->video_capture_controller
.get() == controller
) {
546 void VideoCaptureManager::DestroyDeviceEntryIfNoClients(DeviceEntry
* entry
) {
547 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
548 // Removal of the last client stops the device.
549 if (entry
->video_capture_controller
->GetClientCount() == 0) {
550 DVLOG(1) << "VideoCaptureManager stopping device (type = "
551 << entry
->stream_type
<< ", id = " << entry
->id
<< ")";
553 // The DeviceEntry is removed from |devices_| immediately. The controller is
554 // deleted immediately, and the device is freed asynchronously. After this
555 // point, subsequent requests to open this same device ID will create a new
556 // DeviceEntry, VideoCaptureController, and VideoCaptureDevice.
557 devices_
.erase(entry
);
558 entry
->video_capture_controller
.reset();
559 device_task_runner_
->PostTask(
561 base::Bind(&VideoCaptureManager::DoStopDeviceOnDeviceThread
, this,
562 base::Owned(entry
)));
566 VideoCaptureManager::DeviceEntry
* VideoCaptureManager::GetOrCreateDeviceEntry(
567 media::VideoCaptureSessionId capture_session_id
) {
568 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
570 SessionMap::iterator session_it
= sessions_
.find(capture_session_id
);
571 if (session_it
== sessions_
.end()) {
574 const MediaStreamDevice
& device_info
= session_it
->second
;
576 // Check if another session has already opened this device. If so, just
577 // use that opened device.
578 DeviceEntry
* const existing_device
=
579 GetDeviceEntryForMediaStreamDevice(device_info
);
580 if (existing_device
) {
581 DCHECK_EQ(device_info
.type
, existing_device
->stream_type
);
582 return existing_device
;
585 scoped_ptr
<VideoCaptureController
> video_capture_controller(
586 new VideoCaptureController());
587 DeviceEntry
* new_device
= new DeviceEntry(device_info
.type
,
589 video_capture_controller
.Pass());
590 devices_
.insert(new_device
);
594 VideoCaptureManager::DeviceInfo
* VideoCaptureManager::FindDeviceInfoById(
595 const std::string
& id
,
596 DeviceInfos
& device_vector
) {
597 for (DeviceInfos::iterator it
= device_vector
.begin();
598 it
!= device_vector
.end(); ++it
) {
599 if (it
->name
.id() == id
)
605 void VideoCaptureManager::SetDesktopCaptureWindowIdOnDeviceThread(
607 gfx::NativeViewId window_id
) {
608 DCHECK(IsOnDeviceThread());
609 DCHECK(entry
->stream_type
== MEDIA_DESKTOP_VIDEO_CAPTURE
);
610 #if defined(ENABLE_SCREEN_CAPTURE)
611 DesktopCaptureDevice
* device
=
612 static_cast<DesktopCaptureDevice
*>(entry
->video_capture_device
.get());
613 device
->SetNotificationWindowId(window_id
);
617 void VideoCaptureManager::SaveDesktopCaptureWindowIdOnDeviceThread(
618 media::VideoCaptureSessionId session_id
,
619 gfx::NativeViewId window_id
) {
620 DCHECK(IsOnDeviceThread());
621 DCHECK(notification_window_ids_
.find(session_id
) ==
622 notification_window_ids_
.end());
623 notification_window_ids_
[session_id
] = window_id
;
626 } // namespace content