1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/video/capture/mac/video_capture_device_mac.h"
7 #include <IOKit/IOCFPlugIn.h>
8 #include <IOKit/usb/IOUSBLib.h>
9 #include <IOKit/usb/USBSpec.h>
11 #include "base/bind.h"
12 #include "base/location.h"
13 #include "base/logging.h"
14 #include "base/mac/scoped_ioobject.h"
15 #include "base/mac/scoped_ioplugininterface.h"
16 #include "base/message_loop/message_loop_proxy.h"
17 #include "base/strings/string_number_conversions.h"
18 #include "base/time/time.h"
19 #import "media/base/mac/avfoundation_glue.h"
20 #import "media/video/capture/mac/platform_video_capturing_mac.h"
21 #import "media/video/capture/mac/video_capture_device_avfoundation_mac.h"
22 #import "media/video/capture/mac/video_capture_device_qtkit_mac.h"
23 #include "ui/gfx/geometry/size.h"
25 @implementation DeviceNameAndTransportType
27 - (id)initWithName:(NSString*)deviceName transportType:(int32_t)transportType {
28 if (self = [super init]) {
29 deviceName_.reset([deviceName copy]);
30 transportType_ = transportType;
35 - (NSString*)deviceName {
39 - (int32_t)transportType {
40 return transportType_;
43 @end // @implementation DeviceNameAndTransportType
47 // Mac specific limits for minimum and maximum frame rate.
48 const float kMinFrameRate = 1.0f;
49 const float kMaxFrameRate = 30.0f;
51 // In device identifiers, the USB VID and PID are stored in 4 bytes each.
52 const size_t kVidPidSize = 4;
54 const struct Resolution {
57 } kQVGA = { 320, 240 },
61 const struct Resolution* const kWellSupportedResolutions[] = {
67 // Rescaling the image to fix the pixel aspect ratio runs the risk of making
68 // the aspect ratio worse, if QTKit selects a new source mode with a different
69 // shape. This constant ensures that we don't take this risk if the current
70 // aspect ratio is tolerable.
71 const float kMaxPixelAspectRatio = 1.15;
73 // The following constants are extracted from the specification "Universal
74 // Serial Bus Device Class Definition for Video Devices", Rev. 1.1 June 1, 2005.
75 // http://www.usb.org/developers/devclass_docs/USB_Video_Class_1_1.zip
76 // CS_INTERFACE: Sec. A.4 "Video Class-Specific Descriptor Types".
77 const int kVcCsInterface = 0x24;
78 // VC_PROCESSING_UNIT: Sec. A.5 "Video Class-Specific VC Interface Descriptor
80 const int kVcProcessingUnit = 0x5;
81 // SET_CUR: Sec. A.8 "Video Class-Specific Request Codes".
82 const int kVcRequestCodeSetCur = 0x1;
83 // PU_POWER_LINE_FREQUENCY_CONTROL: Sec. A.9.5 "Processing Unit Control
85 const int kPuPowerLineFrequencyControl = 0x5;
86 // Sec. 4.2.2.3.5 Power Line Frequency Control.
89 const int kPuPowerLineFrequencyControlCommandSize = 1;
91 // Addition to the IOUSB family of structures, with subtype and unit ID.
92 typedef struct IOUSBInterfaceDescriptor {
93 IOUSBDescriptorHeader header;
94 UInt8 bDescriptorSubType;
96 } IOUSBInterfaceDescriptor;
98 static void GetBestMatchSupportedResolution(gfx::Size* resolution) {
99 int min_diff = kint32max;
100 const int desired_area = resolution->GetArea();
101 for (size_t i = 0; i < arraysize(kWellSupportedResolutions); ++i) {
102 const int area = kWellSupportedResolutions[i]->width *
103 kWellSupportedResolutions[i]->height;
104 const int diff = std::abs(desired_area - area);
105 if (diff < min_diff) {
107 resolution->SetSize(kWellSupportedResolutions[i]->width,
108 kWellSupportedResolutions[i]->height);
113 // Tries to create a user-side device interface for a given USB device. Returns
114 // true if interface was found and passes it back in |device_interface|. The
115 // caller should release |device_interface|.
116 static bool FindDeviceInterfaceInUsbDevice(
118 const int product_id,
119 const io_service_t usb_device,
120 IOUSBDeviceInterface*** device_interface) {
121 // Create a plug-in, i.e. a user-side controller to manipulate USB device.
122 IOCFPlugInInterface** plugin;
123 SInt32 score; // Unused, but required for IOCreatePlugInInterfaceForService.
125 IOCreatePlugInInterfaceForService(usb_device,
126 kIOUSBDeviceUserClientTypeID,
127 kIOCFPlugInInterfaceID,
130 if (kr != kIOReturnSuccess || !plugin) {
131 DLOG(ERROR) << "IOCreatePlugInInterfaceForService";
134 base::mac::ScopedIOPluginInterface<IOCFPlugInInterface> plugin_ref(plugin);
136 // Fetch the Device Interface from the plug-in.
138 (*plugin)->QueryInterface(plugin,
139 CFUUIDGetUUIDBytes(kIOUSBDeviceInterfaceID),
140 reinterpret_cast<LPVOID*>(device_interface));
141 if (!SUCCEEDED(res) || !*device_interface) {
142 DLOG(ERROR) << "QueryInterface, couldn't create interface to USB";
148 // Tries to find a Video Control type interface inside a general USB device
149 // interface |device_interface|, and returns it in |video_control_interface| if
150 // found. The returned interface must be released in the caller.
151 static bool FindVideoControlInterfaceInDeviceInterface(
152 IOUSBDeviceInterface** device_interface,
153 IOCFPlugInInterface*** video_control_interface) {
154 // Create an iterator to the list of Video-AVControl interfaces of the device,
155 // then get the first interface in the list.
156 io_iterator_t interface_iterator;
157 IOUSBFindInterfaceRequest interface_request = {
158 .bInterfaceClass = kUSBVideoInterfaceClass,
159 .bInterfaceSubClass = kUSBVideoControlSubClass,
160 .bInterfaceProtocol = kIOUSBFindInterfaceDontCare,
161 .bAlternateSetting = kIOUSBFindInterfaceDontCare
164 (*device_interface)->CreateInterfaceIterator(device_interface,
166 &interface_iterator);
167 if (kr != kIOReturnSuccess) {
168 DLOG(ERROR) << "Could not create an iterator to the device's interfaces.";
171 base::mac::ScopedIOObject<io_iterator_t> iterator_ref(interface_iterator);
173 // There should be just one interface matching the class-subclass desired.
174 io_service_t found_interface;
175 found_interface = IOIteratorNext(interface_iterator);
176 if (!found_interface) {
177 DLOG(ERROR) << "Could not find a Video-AVControl interface in the device.";
180 base::mac::ScopedIOObject<io_service_t> found_interface_ref(found_interface);
182 // Create a user side controller (i.e. a "plug-in") for the found interface.
184 kr = IOCreatePlugInInterfaceForService(found_interface,
185 kIOUSBInterfaceUserClientTypeID,
186 kIOCFPlugInInterfaceID,
187 video_control_interface,
189 if (kr != kIOReturnSuccess || !*video_control_interface) {
190 DLOG(ERROR) << "IOCreatePlugInInterfaceForService";
196 // Creates a control interface for |plugin_interface| and produces a command to
197 // set the appropriate Power Line frequency for flicker removal.
198 static void SetAntiFlickerInVideoControlInterface(
199 IOCFPlugInInterface** plugin_interface,
200 const int frequency) {
201 // Create, the control interface for the found plug-in, and release
202 // the intermediate plug-in.
203 IOUSBInterfaceInterface** control_interface = NULL;
204 HRESULT res = (*plugin_interface)->QueryInterface(
206 CFUUIDGetUUIDBytes(kIOUSBInterfaceInterfaceID),
207 reinterpret_cast<LPVOID*>(&control_interface));
208 if (!SUCCEEDED(res) || !control_interface ) {
209 DLOG(ERROR) << "Couldn’t create control interface";
212 base::mac::ScopedIOPluginInterface<IOUSBInterfaceInterface>
213 control_interface_ref(control_interface);
215 // Find the device's unit ID presenting type 0x24 (kVcCsInterface) and
216 // subtype 0x5 (kVcProcessingUnit). Inside this unit is where we find the
217 // power line frequency removal setting, and this id is device dependent.
218 int real_unit_id = -1;
219 IOUSBDescriptorHeader* descriptor = NULL;
220 IOUSBInterfaceDescriptor* cs_descriptor = NULL;
221 IOUSBInterfaceInterface220** interface =
222 reinterpret_cast<IOUSBInterfaceInterface220**>(control_interface);
223 while ((descriptor = (*interface)->FindNextAssociatedDescriptor(
224 interface, descriptor, kUSBAnyDesc))) {
226 reinterpret_cast<IOUSBInterfaceDescriptor*>(descriptor);
227 if ((descriptor->bDescriptorType == kVcCsInterface) &&
228 (cs_descriptor->bDescriptorSubType == kVcProcessingUnit)) {
229 real_unit_id = cs_descriptor->bUnitID;
233 DVLOG_IF(1, real_unit_id == -1) << "This USB device doesn't seem to have a "
234 << " VC_PROCESSING_UNIT, anti-flicker not available";
235 if (real_unit_id == -1)
238 if ((*control_interface)->USBInterfaceOpen(control_interface) !=
240 DLOG(ERROR) << "Unable to open control interface";
244 // Create the control request and launch it to the device's control interface.
245 // Note how the wIndex needs the interface number OR'ed in the lowest bits.
246 IOUSBDevRequest command;
247 command.bmRequestType = USBmakebmRequestType(kUSBOut,
250 command.bRequest = kVcRequestCodeSetCur;
251 UInt8 interface_number;
252 (*control_interface)->GetInterfaceNumber(control_interface,
254 command.wIndex = (real_unit_id << 8) | interface_number;
255 const int selector = kPuPowerLineFrequencyControl;
256 command.wValue = (selector << 8);
257 command.wLength = kPuPowerLineFrequencyControlCommandSize;
258 command.wLenDone = 0;
259 int power_line_flag_value = (frequency == 50) ? k50Hz : k60Hz;
260 command.pData = &power_line_flag_value;
262 IOReturn ret = (*control_interface)->ControlRequest(control_interface,
264 DLOG_IF(ERROR, ret != kIOReturnSuccess) << "Anti-flicker control request"
265 << " failed (0x" << std::hex << ret << "), unit id: " << real_unit_id;
266 DVLOG_IF(1, ret == kIOReturnSuccess) << "Anti-flicker set to " << frequency
269 (*control_interface)->USBInterfaceClose(control_interface);
272 // Sets the flicker removal in a USB webcam identified by |vendor_id| and
273 // |product_id|, if available. The process includes first finding all USB
274 // devices matching the specified |vendor_id| and |product_id|; for each
275 // matching device, a device interface, and inside it a video control interface
276 // are created. The latter is used to a send a power frequency setting command.
277 static void SetAntiFlickerInUsbDevice(const int vendor_id,
278 const int product_id,
279 const int frequency) {
282 DVLOG(1) << "Setting Power Line Frequency to " << frequency << " Hz, device "
283 << std::hex << vendor_id << "-" << product_id;
285 // Compose a search dictionary with vendor and product ID.
286 CFMutableDictionaryRef query_dictionary =
287 IOServiceMatching(kIOUSBDeviceClassName);
288 CFDictionarySetValue(query_dictionary, CFSTR(kUSBVendorName),
289 CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &vendor_id));
290 CFDictionarySetValue(query_dictionary, CFSTR(kUSBProductName),
291 CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &product_id));
293 io_iterator_t usb_iterator;
294 kern_return_t kr = IOServiceGetMatchingServices(kIOMasterPortDefault,
297 if (kr != kIOReturnSuccess) {
298 DLOG(ERROR) << "No devices found with specified Vendor and Product ID.";
301 base::mac::ScopedIOObject<io_iterator_t> usb_iterator_ref(usb_iterator);
303 while (io_service_t usb_device = IOIteratorNext(usb_iterator)) {
304 base::mac::ScopedIOObject<io_service_t> usb_device_ref(usb_device);
306 IOUSBDeviceInterface** device_interface = NULL;
307 if (!FindDeviceInterfaceInUsbDevice(vendor_id, product_id,
308 usb_device, &device_interface)) {
311 base::mac::ScopedIOPluginInterface<IOUSBDeviceInterface>
312 device_interface_ref(device_interface);
314 IOCFPlugInInterface** video_control_interface = NULL;
315 if (!FindVideoControlInterfaceInDeviceInterface(device_interface,
316 &video_control_interface)) {
319 base::mac::ScopedIOPluginInterface<IOCFPlugInInterface>
320 plugin_interface_ref(video_control_interface);
322 SetAntiFlickerInVideoControlInterface(video_control_interface, frequency);
326 const std::string VideoCaptureDevice::Name::GetModel() const {
327 // Skip the AVFoundation's not USB nor built-in devices.
328 if (capture_api_type() == AVFOUNDATION && transport_type() != USB_OR_BUILT_IN)
330 if (capture_api_type() == DECKLINK)
332 // Both PID and VID are 4 characters.
333 if (unique_id_.size() < 2 * kVidPidSize)
336 // The last characters of device id is a concatenation of VID and then PID.
337 const size_t vid_location = unique_id_.size() - 2 * kVidPidSize;
338 std::string id_vendor = unique_id_.substr(vid_location, kVidPidSize);
339 const size_t pid_location = unique_id_.size() - kVidPidSize;
340 std::string id_product = unique_id_.substr(pid_location, kVidPidSize);
342 return id_vendor + ":" + id_product;
345 VideoCaptureDeviceMac::VideoCaptureDeviceMac(const Name& device_name)
346 : device_name_(device_name),
347 tried_to_square_pixels_(false),
348 task_runner_(base::MessageLoopProxy::current()),
349 state_(kNotInitialized),
350 capture_device_(nil),
351 weak_factory_(this) {
352 // Avoid reconfiguring AVFoundation or blacklisted devices.
353 final_resolution_selected_ = AVFoundationGlue::IsAVFoundationSupported() ||
354 device_name.is_blacklisted();
357 VideoCaptureDeviceMac::~VideoCaptureDeviceMac() {
358 DCHECK(task_runner_->BelongsToCurrentThread());
359 [capture_device_ release];
362 void VideoCaptureDeviceMac::AllocateAndStart(
363 const VideoCaptureParams& params,
364 scoped_ptr<VideoCaptureDevice::Client> client) {
365 DCHECK(task_runner_->BelongsToCurrentThread());
366 if (state_ != kIdle) {
370 // QTKit API can scale captured frame to any size requested, which would lead
371 // to undesired aspect ratio changes. Try to open the camera with a known
372 // supported format and let the client crop/pad the captured frames.
373 gfx::Size resolution = params.requested_format.frame_size;
374 if (!AVFoundationGlue::IsAVFoundationSupported())
375 GetBestMatchSupportedResolution(&resolution);
377 client_ = client.Pass();
378 if (device_name_.capture_api_type() == Name::AVFOUNDATION)
379 LogMessage("Using AVFoundation for device: " + device_name_.name());
381 LogMessage("Using QTKit for device: " + device_name_.name());
383 [NSString stringWithUTF8String:device_name_.id().c_str()];
385 [capture_device_ setFrameReceiver:this];
387 if (![capture_device_ setCaptureDevice:deviceId]) {
388 SetErrorState("Could not open capture device.");
392 capture_format_.frame_size = resolution;
393 capture_format_.frame_rate =
394 std::max(kMinFrameRate,
395 std::min(params.requested_format.frame_rate, kMaxFrameRate));
396 // Leave the pixel format selection to AVFoundation/QTKit. The pixel format
397 // will be passed to |ReceiveFrame|.
398 capture_format_.pixel_format = PIXEL_FORMAT_UNKNOWN;
400 // QTKit: Set the capture resolution only if this is VGA or smaller, otherwise
401 // leave it unconfigured and start capturing: QTKit will produce frames at the
402 // native resolution, allowing us to identify cameras whose native resolution
403 // is too low for HD. This additional information comes at a cost in startup
404 // latency, because the webcam will need to be reopened if its default
405 // resolution is not HD or VGA.
406 // AVfoundation is configured for all resolutions.
407 if (AVFoundationGlue::IsAVFoundationSupported() ||
408 resolution.width() <= kVGA.width || resolution.height() <= kVGA.height) {
409 if (!UpdateCaptureResolution())
413 // Try setting the power line frequency removal (anti-flicker). The built-in
414 // cameras are normally suspended so the configuration must happen right
415 // before starting capture and during configuration.
416 const std::string& device_model = device_name_.GetModel();
417 if (device_model.length() > 2 * kVidPidSize) {
418 std::string vendor_id = device_model.substr(0, kVidPidSize);
419 std::string model_id = device_model.substr(kVidPidSize + 1);
420 int vendor_id_as_int, model_id_as_int;
421 if (base::HexStringToInt(base::StringPiece(vendor_id), &vendor_id_as_int) &&
422 base::HexStringToInt(base::StringPiece(model_id), &model_id_as_int)) {
423 SetAntiFlickerInUsbDevice(vendor_id_as_int, model_id_as_int,
424 GetPowerLineFrequencyForLocation());
428 if (![capture_device_ startCapture]) {
429 SetErrorState("Could not start capture device.");
436 void VideoCaptureDeviceMac::StopAndDeAllocate() {
437 DCHECK(task_runner_->BelongsToCurrentThread());
438 DCHECK(state_ == kCapturing || state_ == kError) << state_;
440 [capture_device_ setCaptureDevice:nil];
441 [capture_device_ setFrameReceiver:nil];
444 tried_to_square_pixels_ = false;
447 bool VideoCaptureDeviceMac::Init(
448 VideoCaptureDevice::Name::CaptureApiType capture_api_type) {
449 DCHECK(task_runner_->BelongsToCurrentThread());
450 DCHECK_EQ(state_, kNotInitialized);
452 if (capture_api_type == Name::AVFOUNDATION) {
454 [[VideoCaptureDeviceAVFoundation alloc] initWithFrameReceiver:this];
457 [[VideoCaptureDeviceQTKit alloc] initWithFrameReceiver:this];
460 if (!capture_device_)
467 void VideoCaptureDeviceMac::ReceiveFrame(
468 const uint8* video_frame,
469 int video_frame_length,
470 const VideoCaptureFormat& frame_format,
471 int aspect_numerator,
472 int aspect_denominator) {
473 // This method is safe to call from a device capture thread, i.e. any thread
474 // controlled by QTKit/AVFoundation.
475 if (!final_resolution_selected_) {
476 DCHECK(!AVFoundationGlue::IsAVFoundationSupported());
477 if (capture_format_.frame_size.width() > kVGA.width ||
478 capture_format_.frame_size.height() > kVGA.height) {
479 // We are requesting HD. Make sure that the picture is good, otherwise
481 bool change_to_vga = false;
482 if (frame_format.frame_size.width() <
483 capture_format_.frame_size.width() ||
484 frame_format.frame_size.height() <
485 capture_format_.frame_size.height()) {
486 // These are the default capture settings, not yet configured to match
487 // |capture_format_|.
488 DCHECK(frame_format.frame_rate == 0);
489 DVLOG(1) << "Switching to VGA because the default resolution is " <<
490 frame_format.frame_size.ToString();
491 change_to_vga = true;
494 if (capture_format_.frame_size == frame_format.frame_size &&
495 aspect_numerator != aspect_denominator) {
496 DVLOG(1) << "Switching to VGA because HD has nonsquare pixel " <<
497 "aspect ratio " << aspect_numerator << ":" << aspect_denominator;
498 change_to_vga = true;
502 capture_format_.frame_size.SetSize(kVGA.width, kVGA.height);
505 if (capture_format_.frame_size == frame_format.frame_size &&
506 !tried_to_square_pixels_ &&
507 (aspect_numerator > kMaxPixelAspectRatio * aspect_denominator ||
508 aspect_denominator > kMaxPixelAspectRatio * aspect_numerator)) {
509 // The requested size results in non-square PAR. Shrink the frame to 1:1
510 // PAR (assuming QTKit selects the same input mode, which is not
512 int new_width = capture_format_.frame_size.width();
513 int new_height = capture_format_.frame_size.height();
514 if (aspect_numerator < aspect_denominator)
515 new_width = (new_width * aspect_numerator) / aspect_denominator;
517 new_height = (new_height * aspect_denominator) / aspect_numerator;
518 capture_format_.frame_size.SetSize(new_width, new_height);
519 tried_to_square_pixels_ = true;
522 if (capture_format_.frame_size == frame_format.frame_size) {
523 final_resolution_selected_ = true;
525 UpdateCaptureResolution();
526 // Let the resolution update sink through QTKit and wait for next frame.
531 // QTKit capture source can change resolution if someone else reconfigures the
532 // camera, and that is fine: http://crbug.com/353620. In AVFoundation, this
533 // should not happen, it should resize internally.
534 if (!AVFoundationGlue::IsAVFoundationSupported()) {
535 capture_format_.frame_size = frame_format.frame_size;
536 } else if (capture_format_.frame_size != frame_format.frame_size) {
537 ReceiveError("Captured resolution " + frame_format.frame_size.ToString() +
538 ", and expected " + capture_format_.frame_size.ToString());
542 client_->OnIncomingCapturedData(video_frame,
546 base::TimeTicks::Now());
549 void VideoCaptureDeviceMac::ReceiveError(const std::string& reason) {
550 task_runner_->PostTask(FROM_HERE,
551 base::Bind(&VideoCaptureDeviceMac::SetErrorState,
552 weak_factory_.GetWeakPtr(),
556 void VideoCaptureDeviceMac::SetErrorState(const std::string& reason) {
557 DCHECK(task_runner_->BelongsToCurrentThread());
559 client_->OnError(reason);
562 void VideoCaptureDeviceMac::LogMessage(const std::string& message) {
563 DCHECK(task_runner_->BelongsToCurrentThread());
565 client_->OnLog(message);
568 bool VideoCaptureDeviceMac::UpdateCaptureResolution() {
569 if (![capture_device_ setCaptureHeight:capture_format_.frame_size.height()
570 width:capture_format_.frame_size.width()
571 frameRate:capture_format_.frame_rate]) {
572 ReceiveError("Could not configure capture device.");