1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/video/capture/mac/video_capture_device_mac.h"
7 #include <IOKit/IOCFPlugIn.h>
8 #include <IOKit/usb/IOUSBLib.h>
9 #include <IOKit/usb/USBSpec.h>
11 #include "base/bind.h"
12 #include "base/location.h"
13 #include "base/logging.h"
14 #include "base/message_loop/message_loop_proxy.h"
15 #include "base/mac/scoped_ioobject.h"
16 #include "base/mac/scoped_ioplugininterface.h"
17 #include "base/strings/string_number_conversions.h"
18 #include "base/time/time.h"
19 #import "media/video/capture/mac/avfoundation_glue.h"
20 #import "media/video/capture/mac/platform_video_capturing_mac.h"
21 #import "media/video/capture/mac/video_capture_device_avfoundation_mac.h"
22 #import "media/video/capture/mac/video_capture_device_qtkit_mac.h"
26 const int kMinFrameRate = 1;
27 const int kMaxFrameRate = 30;
29 // In device identifiers, the USB VID and PID are stored in 4 bytes each.
30 const size_t kVidPidSize = 4;
32 const struct Resolution {
35 } kQVGA = { 320, 240 },
39 const struct Resolution* const kWellSupportedResolutions[] = {
45 // Rescaling the image to fix the pixel aspect ratio runs the risk of making
46 // the aspect ratio worse, if QTKit selects a new source mode with a different
47 // shape. This constant ensures that we don't take this risk if the current
48 // aspect ratio is tolerable.
49 const float kMaxPixelAspectRatio = 1.15;
51 // The following constants are extracted from the specification "Universal
52 // Serial Bus Device Class Definition for Video Devices", Rev. 1.1 June 1, 2005.
53 // http://www.usb.org/developers/devclass_docs/USB_Video_Class_1_1.zip
54 // CS_INTERFACE: Sec. A.4 "Video Class-Specific Descriptor Types".
55 const int kVcCsInterface = 0x24;
56 // VC_PROCESSING_UNIT: Sec. A.5 "Video Class-Specific VC Interface Descriptor
58 const int kVcProcessingUnit = 0x5;
59 // SET_CUR: Sec. A.8 "Video Class-Specific Request Codes".
60 const int kVcRequestCodeSetCur = 0x1;
61 // PU_POWER_LINE_FREQUENCY_CONTROL: Sec. A.9.5 "Processing Unit Control
63 const int kPuPowerLineFrequencyControl = 0x5;
64 // Sec. 4.2.2.3.5 Power Line Frequency Control.
67 const int kPuPowerLineFrequencyControlCommandSize = 1;
69 // Addition to the IOUSB family of structures, with subtype and unit ID.
70 typedef struct IOUSBInterfaceDescriptor {
71 IOUSBDescriptorHeader header;
72 UInt8 bDescriptorSubType;
74 } IOUSBInterfaceDescriptor;
76 // TODO(ronghuawu): Replace this with CapabilityList::GetBestMatchedCapability.
77 void GetBestMatchSupportedResolution(int* width, int* height) {
78 int min_diff = kint32max;
79 int matched_width = *width;
80 int matched_height = *height;
81 int desired_res_area = *width * *height;
82 for (size_t i = 0; i < arraysize(kWellSupportedResolutions); ++i) {
83 int area = kWellSupportedResolutions[i]->width *
84 kWellSupportedResolutions[i]->height;
85 int diff = std::abs(desired_res_area - area);
86 if (diff < min_diff) {
88 matched_width = kWellSupportedResolutions[i]->width;
89 matched_height = kWellSupportedResolutions[i]->height;
92 *width = matched_width;
93 *height = matched_height;
96 // Tries to create a user-side device interface for a given USB device. Returns
97 // true if interface was found and passes it back in |device_interface|. The
98 // caller should release |device_interface|.
99 static bool FindDeviceInterfaceInUsbDevice(
101 const int product_id,
102 const io_service_t usb_device,
103 IOUSBDeviceInterface*** device_interface) {
104 // Create a plug-in, i.e. a user-side controller to manipulate USB device.
105 IOCFPlugInInterface** plugin;
106 SInt32 score; // Unused, but required for IOCreatePlugInInterfaceForService.
108 IOCreatePlugInInterfaceForService(usb_device,
109 kIOUSBDeviceUserClientTypeID,
110 kIOCFPlugInInterfaceID,
113 if (kr != kIOReturnSuccess || !plugin) {
114 DLOG(ERROR) << "IOCreatePlugInInterfaceForService";
117 base::mac::ScopedIOPluginInterface<IOCFPlugInInterface> plugin_ref(plugin);
119 // Fetch the Device Interface from the plug-in.
121 (*plugin)->QueryInterface(plugin,
122 CFUUIDGetUUIDBytes(kIOUSBDeviceInterfaceID),
123 reinterpret_cast<LPVOID*>(device_interface));
124 if (!SUCCEEDED(res) || !*device_interface) {
125 DLOG(ERROR) << "QueryInterface, couldn't create interface to USB";
131 // Tries to find a Video Control type interface inside a general USB device
132 // interface |device_interface|, and returns it in |video_control_interface| if
133 // found. The returned interface must be released in the caller.
134 static bool FindVideoControlInterfaceInDeviceInterface(
135 IOUSBDeviceInterface** device_interface,
136 IOCFPlugInInterface*** video_control_interface) {
137 // Create an iterator to the list of Video-AVControl interfaces of the device,
138 // then get the first interface in the list.
139 io_iterator_t interface_iterator;
140 IOUSBFindInterfaceRequest interface_request = {
141 .bInterfaceClass = kUSBVideoInterfaceClass,
142 .bInterfaceSubClass = kUSBVideoControlSubClass,
143 .bInterfaceProtocol = kIOUSBFindInterfaceDontCare,
144 .bAlternateSetting = kIOUSBFindInterfaceDontCare
147 (*device_interface)->CreateInterfaceIterator(device_interface,
149 &interface_iterator);
150 if (kr != kIOReturnSuccess) {
151 DLOG(ERROR) << "Could not create an iterator to the device's interfaces.";
154 base::mac::ScopedIOObject<io_iterator_t> iterator_ref(interface_iterator);
156 // There should be just one interface matching the class-subclass desired.
157 io_service_t found_interface;
158 found_interface = IOIteratorNext(interface_iterator);
159 if (!found_interface) {
160 DLOG(ERROR) << "Could not find a Video-AVControl interface in the device.";
163 base::mac::ScopedIOObject<io_service_t> found_interface_ref(found_interface);
165 // Create a user side controller (i.e. a "plug-in") for the found interface.
167 kr = IOCreatePlugInInterfaceForService(found_interface,
168 kIOUSBInterfaceUserClientTypeID,
169 kIOCFPlugInInterfaceID,
170 video_control_interface,
172 if (kr != kIOReturnSuccess || !*video_control_interface) {
173 DLOG(ERROR) << "IOCreatePlugInInterfaceForService";
179 // Creates a control interface for |plugin_interface| and produces a command to
180 // set the appropriate Power Line frequency for flicker removal.
181 static void SetAntiFlickerInVideoControlInterface(
182 IOCFPlugInInterface** plugin_interface,
183 const int frequency) {
184 // Create, the control interface for the found plug-in, and release
185 // the intermediate plug-in.
186 IOUSBInterfaceInterface** control_interface = NULL;
187 HRESULT res = (*plugin_interface)->QueryInterface(
189 CFUUIDGetUUIDBytes(kIOUSBInterfaceInterfaceID),
190 reinterpret_cast<LPVOID*>(&control_interface));
191 if (!SUCCEEDED(res) || !control_interface ) {
192 DLOG(ERROR) << "Couldn’t create control interface";
195 base::mac::ScopedIOPluginInterface<IOUSBInterfaceInterface>
196 control_interface_ref(control_interface);
198 // Find the device's unit ID presenting type 0x24 (kVcCsInterface) and
199 // subtype 0x5 (kVcProcessingUnit). Inside this unit is where we find the
200 // power line frequency removal setting, and this id is device dependent.
201 int real_unit_id = -1;
202 IOUSBDescriptorHeader* descriptor = NULL;
203 IOUSBInterfaceDescriptor* cs_descriptor = NULL;
204 IOUSBInterfaceInterface220** interface =
205 reinterpret_cast<IOUSBInterfaceInterface220**>(control_interface);
206 while ((descriptor = (*interface)->FindNextAssociatedDescriptor(
207 interface, descriptor, kUSBAnyDesc))) {
209 reinterpret_cast<IOUSBInterfaceDescriptor*>(descriptor);
210 if ((descriptor->bDescriptorType == kVcCsInterface) &&
211 (cs_descriptor->bDescriptorSubType == kVcProcessingUnit)) {
212 real_unit_id = cs_descriptor->bUnitID;
216 DVLOG_IF(1, real_unit_id == -1) << "This USB device doesn't seem to have a "
217 << " VC_PROCESSING_UNIT, anti-flicker not available";
218 if (real_unit_id == -1)
221 if ((*control_interface)->USBInterfaceOpen(control_interface) !=
223 DLOG(ERROR) << "Unable to open control interface";
227 // Create the control request and launch it to the device's control interface.
228 // Note how the wIndex needs the interface number OR'ed in the lowest bits.
229 IOUSBDevRequest command;
230 command.bmRequestType = USBmakebmRequestType(kUSBOut,
233 command.bRequest = kVcRequestCodeSetCur;
234 UInt8 interface_number;
235 (*control_interface)->GetInterfaceNumber(control_interface,
237 command.wIndex = (real_unit_id << 8) | interface_number;
238 const int selector = kPuPowerLineFrequencyControl;
239 command.wValue = (selector << 8);
240 command.wLength = kPuPowerLineFrequencyControlCommandSize;
241 command.wLenDone = 0;
242 int power_line_flag_value = (frequency == 50) ? k50Hz : k60Hz;
243 command.pData = &power_line_flag_value;
245 IOReturn ret = (*control_interface)->ControlRequest(control_interface,
247 DLOG_IF(ERROR, ret != kIOReturnSuccess) << "Anti-flicker control request"
248 << " failed (0x" << std::hex << ret << "), unit id: " << real_unit_id;
249 DVLOG_IF(1, ret == kIOReturnSuccess) << "Anti-flicker set to " << frequency
252 (*control_interface)->USBInterfaceClose(control_interface);
255 // Sets the flicker removal in a USB webcam identified by |vendor_id| and
256 // |product_id|, if available. The process includes first finding all USB
257 // devices matching the specified |vendor_id| and |product_id|; for each
258 // matching device, a device interface, and inside it a video control interface
259 // are created. The latter is used to a send a power frequency setting command.
260 static void SetAntiFlickerInUsbDevice(const int vendor_id,
261 const int product_id,
262 const int frequency) {
265 DVLOG(1) << "Setting Power Line Frequency to " << frequency << " Hz, device "
266 << std::hex << vendor_id << "-" << product_id;
268 // Compose a search dictionary with vendor and product ID.
269 CFMutableDictionaryRef query_dictionary =
270 IOServiceMatching(kIOUSBDeviceClassName);
271 CFDictionarySetValue(query_dictionary, CFSTR(kUSBVendorName),
272 CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &vendor_id));
273 CFDictionarySetValue(query_dictionary, CFSTR(kUSBProductName),
274 CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &product_id));
276 io_iterator_t usb_iterator;
277 kern_return_t kr = IOServiceGetMatchingServices(kIOMasterPortDefault,
280 if (kr != kIOReturnSuccess) {
281 DLOG(ERROR) << "No devices found with specified Vendor and Product ID.";
284 base::mac::ScopedIOObject<io_iterator_t> usb_iterator_ref(usb_iterator);
286 while (io_service_t usb_device = IOIteratorNext(usb_iterator)) {
287 base::mac::ScopedIOObject<io_service_t> usb_device_ref(usb_device);
289 IOUSBDeviceInterface** device_interface = NULL;
290 if (!FindDeviceInterfaceInUsbDevice(vendor_id, product_id,
291 usb_device, &device_interface)) {
294 base::mac::ScopedIOPluginInterface<IOUSBDeviceInterface>
295 device_interface_ref(device_interface);
297 IOCFPlugInInterface** video_control_interface = NULL;
298 if (!FindVideoControlInterfaceInDeviceInterface(device_interface,
299 &video_control_interface)) {
302 base::mac::ScopedIOPluginInterface<IOCFPlugInInterface>
303 plugin_interface_ref(video_control_interface);
305 SetAntiFlickerInVideoControlInterface(video_control_interface, frequency);
309 // TODO(mcasas): Remove the following static methods when they are no longer
310 // referenced from VideoCaptureDeviceFactory, i.e. when all OS platforms have
311 // splitted the VideoCaptureDevice into VideoCaptureDevice and
312 // VideoCaptureDeviceFactory.
315 VideoCaptureDevice* VideoCaptureDevice::Create(
316 scoped_refptr<base::SingleThreadTaskRunner> ui_task_runner,
317 const Name& device_name) {
322 void VideoCaptureDevice::GetDeviceNames(Names* device_names) {
327 void VideoCaptureDevice::GetDeviceSupportedFormats(
329 VideoCaptureFormats* supported_formats) {
333 const std::string VideoCaptureDevice::Name::GetModel() const {
334 // Both PID and VID are 4 characters.
335 if (unique_id_.size() < 2 * kVidPidSize) {
339 // The last characters of device id is a concatenation of VID and then PID.
340 const size_t vid_location = unique_id_.size() - 2 * kVidPidSize;
341 std::string id_vendor = unique_id_.substr(vid_location, kVidPidSize);
342 const size_t pid_location = unique_id_.size() - kVidPidSize;
343 std::string id_product = unique_id_.substr(pid_location, kVidPidSize);
345 return id_vendor + ":" + id_product;
348 VideoCaptureDeviceMac::VideoCaptureDeviceMac(const Name& device_name)
349 : device_name_(device_name),
350 tried_to_square_pixels_(false),
351 task_runner_(base::MessageLoopProxy::current()),
352 state_(kNotInitialized),
353 capture_device_(nil),
354 weak_factory_(this) {
355 final_resolution_selected_ = AVFoundationGlue::IsAVFoundationSupported();
358 VideoCaptureDeviceMac::~VideoCaptureDeviceMac() {
359 DCHECK(task_runner_->BelongsToCurrentThread());
360 [capture_device_ release];
363 void VideoCaptureDeviceMac::AllocateAndStart(
364 const VideoCaptureParams& params,
365 scoped_ptr<VideoCaptureDevice::Client> client) {
366 DCHECK(task_runner_->BelongsToCurrentThread());
367 if (state_ != kIdle) {
370 int width = params.requested_format.frame_size.width();
371 int height = params.requested_format.frame_size.height();
372 int frame_rate = params.requested_format.frame_rate;
374 // QTKit API can scale captured frame to any size requested, which would lead
375 // to undesired aspect ratio changes. Try to open the camera with a known
376 // supported format and let the client crop/pad the captured frames.
377 if (!AVFoundationGlue::IsAVFoundationSupported())
378 GetBestMatchSupportedResolution(&width, &height);
380 client_ = client.Pass();
381 if (device_name_.capture_api_type() == Name::AVFOUNDATION)
382 LogMessage("Using AVFoundation for device: " + device_name_.name());
384 LogMessage("Using QTKit for device: " + device_name_.name());
386 [NSString stringWithUTF8String:device_name_.id().c_str()];
388 [capture_device_ setFrameReceiver:this];
390 if (![capture_device_ setCaptureDevice:deviceId]) {
391 SetErrorState("Could not open capture device.");
394 if (frame_rate < kMinFrameRate)
395 frame_rate = kMinFrameRate;
396 else if (frame_rate > kMaxFrameRate)
397 frame_rate = kMaxFrameRate;
399 capture_format_.frame_size.SetSize(width, height);
400 capture_format_.frame_rate = frame_rate;
401 capture_format_.pixel_format = PIXEL_FORMAT_UYVY;
403 // QTKit: Set the capture resolution only if this is VGA or smaller, otherwise
404 // leave it unconfigured and start capturing: QTKit will produce frames at the
405 // native resolution, allowing us to identify cameras whose native resolution
406 // is too low for HD. This additional information comes at a cost in startup
407 // latency, because the webcam will need to be reopened if its default
408 // resolution is not HD or VGA.
409 // AVfoundation is configured for all resolutions.
410 if (AVFoundationGlue::IsAVFoundationSupported() || width <= kVGA.width ||
411 height <= kVGA.height) {
412 if (!UpdateCaptureResolution())
416 // Try setting the power line frequency removal (anti-flicker). The built-in
417 // cameras are normally suspended so the configuration must happen right
418 // before starting capture and during configuration.
419 const std::string& device_model = device_name_.GetModel();
420 if (device_model.length() > 2 * kVidPidSize) {
421 std::string vendor_id = device_model.substr(0, kVidPidSize);
422 std::string model_id = device_model.substr(kVidPidSize + 1);
423 int vendor_id_as_int, model_id_as_int;
424 if (base::HexStringToInt(base::StringPiece(vendor_id), &vendor_id_as_int) &&
425 base::HexStringToInt(base::StringPiece(model_id), &model_id_as_int)) {
426 SetAntiFlickerInUsbDevice(vendor_id_as_int, model_id_as_int,
427 GetPowerLineFrequencyForLocation());
431 if (![capture_device_ startCapture]) {
432 SetErrorState("Could not start capture device.");
439 void VideoCaptureDeviceMac::StopAndDeAllocate() {
440 DCHECK(task_runner_->BelongsToCurrentThread());
441 DCHECK(state_ == kCapturing || state_ == kError) << state_;
442 [capture_device_ stopCapture];
444 [capture_device_ setCaptureDevice:nil];
445 [capture_device_ setFrameReceiver:nil];
448 tried_to_square_pixels_ = false;
451 bool VideoCaptureDeviceMac::Init(
452 VideoCaptureDevice::Name::CaptureApiType capture_api_type) {
453 DCHECK(task_runner_->BelongsToCurrentThread());
454 DCHECK_EQ(state_, kNotInitialized);
456 if (capture_api_type == Name::AVFOUNDATION) {
458 [[VideoCaptureDeviceAVFoundation alloc] initWithFrameReceiver:this];
461 [[VideoCaptureDeviceQTKit alloc] initWithFrameReceiver:this];
464 if (!capture_device_)
471 void VideoCaptureDeviceMac::ReceiveFrame(
472 const uint8* video_frame,
473 int video_frame_length,
474 const VideoCaptureFormat& frame_format,
475 int aspect_numerator,
476 int aspect_denominator) {
477 // This method is safe to call from a device capture thread, i.e. any thread
478 // controlled by QTKit/AVFoundation.
479 if (!final_resolution_selected_) {
480 DCHECK(!AVFoundationGlue::IsAVFoundationSupported());
481 if (capture_format_.frame_size.width() > kVGA.width ||
482 capture_format_.frame_size.height() > kVGA.height) {
483 // We are requesting HD. Make sure that the picture is good, otherwise
485 bool change_to_vga = false;
486 if (frame_format.frame_size.width() <
487 capture_format_.frame_size.width() ||
488 frame_format.frame_size.height() <
489 capture_format_.frame_size.height()) {
490 // These are the default capture settings, not yet configured to match
491 // |capture_format_|.
492 DCHECK(frame_format.frame_rate == 0);
493 DVLOG(1) << "Switching to VGA because the default resolution is " <<
494 frame_format.frame_size.ToString();
495 change_to_vga = true;
498 if (capture_format_.frame_size == frame_format.frame_size &&
499 aspect_numerator != aspect_denominator) {
500 DVLOG(1) << "Switching to VGA because HD has nonsquare pixel " <<
501 "aspect ratio " << aspect_numerator << ":" << aspect_denominator;
502 change_to_vga = true;
506 capture_format_.frame_size.SetSize(kVGA.width, kVGA.height);
509 if (capture_format_.frame_size == frame_format.frame_size &&
510 !tried_to_square_pixels_ &&
511 (aspect_numerator > kMaxPixelAspectRatio * aspect_denominator ||
512 aspect_denominator > kMaxPixelAspectRatio * aspect_numerator)) {
513 // The requested size results in non-square PAR. Shrink the frame to 1:1
514 // PAR (assuming QTKit selects the same input mode, which is not
516 int new_width = capture_format_.frame_size.width();
517 int new_height = capture_format_.frame_size.height();
518 if (aspect_numerator < aspect_denominator)
519 new_width = (new_width * aspect_numerator) / aspect_denominator;
521 new_height = (new_height * aspect_denominator) / aspect_numerator;
522 capture_format_.frame_size.SetSize(new_width, new_height);
523 tried_to_square_pixels_ = true;
526 if (capture_format_.frame_size == frame_format.frame_size) {
527 final_resolution_selected_ = true;
529 UpdateCaptureResolution();
530 // Let the resolution update sink through QTKit and wait for next frame.
535 // QTKit capture source can change resolution if someone else reconfigures the
536 // camera, and that is fine: http://crbug.com/353620. In AVFoundation, this
537 // should not happen, it should resize internally.
538 if (!AVFoundationGlue::IsAVFoundationSupported()) {
539 capture_format_.frame_size = frame_format.frame_size;
540 } else if (capture_format_.frame_size != frame_format.frame_size) {
541 ReceiveError("Captured resolution " + frame_format.frame_size.ToString() +
542 ", and expected " + capture_format_.frame_size.ToString());
546 client_->OnIncomingCapturedData(video_frame,
550 base::TimeTicks::Now());
553 void VideoCaptureDeviceMac::ReceiveError(const std::string& reason) {
554 task_runner_->PostTask(FROM_HERE,
555 base::Bind(&VideoCaptureDeviceMac::SetErrorState,
556 weak_factory_.GetWeakPtr(),
560 void VideoCaptureDeviceMac::SetErrorState(const std::string& reason) {
561 DCHECK(task_runner_->BelongsToCurrentThread());
562 DLOG(ERROR) << reason;
564 client_->OnError(reason);
567 void VideoCaptureDeviceMac::LogMessage(const std::string& message) {
568 DCHECK(task_runner_->BelongsToCurrentThread());
570 client_->OnLog(message);
573 bool VideoCaptureDeviceMac::UpdateCaptureResolution() {
574 if (![capture_device_ setCaptureHeight:capture_format_.frame_size.height()
575 width:capture_format_.frame_size.width()
576 frameRate:capture_format_.frame_rate]) {
577 ReceiveError("Could not configure capture device.");