1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/capture/video/mac/video_capture_device_mac.h"
7 #include <IOKit/IOCFPlugIn.h>
8 #include <IOKit/usb/IOUSBLib.h>
9 #include <IOKit/usb/USBSpec.h>
11 #include "base/bind.h"
12 #include "base/location.h"
13 #include "base/logging.h"
14 #include "base/mac/scoped_ioobject.h"
15 #include "base/mac/scoped_ioplugininterface.h"
16 #include "base/single_thread_task_runner.h"
17 #include "base/strings/string_number_conversions.h"
18 #include "base/thread_task_runner_handle.h"
19 #include "base/time/time.h"
20 #import "media/base/mac/avfoundation_glue.h"
21 #import "media/capture/video/mac/platform_video_capturing_mac.h"
22 #import "media/capture/video/mac/video_capture_device_avfoundation_mac.h"
23 #import "media/capture/video/mac/video_capture_device_qtkit_mac.h"
24 #include "ui/gfx/geometry/size.h"
26 @implementation DeviceNameAndTransportType
28 - (id)initWithName:(NSString*)deviceName transportType:(int32_t)transportType {
29 if (self = [super init]) {
30 deviceName_.reset([deviceName copy]);
31 transportType_ = transportType;
36 - (NSString*)deviceName {
40 - (int32_t)transportType {
41 return transportType_;
44 @end // @implementation DeviceNameAndTransportType
48 // Mac specific limits for minimum and maximum frame rate.
49 const float kMinFrameRate = 1.0f;
50 const float kMaxFrameRate = 30.0f;
52 // In device identifiers, the USB VID and PID are stored in 4 bytes each.
53 const size_t kVidPidSize = 4;
55 const struct Resolution {
58 } kQVGA = {320, 240}, kVGA = {640, 480}, kHD = {1280, 720};
60 const struct Resolution* const kWellSupportedResolutions[] = {
66 // Rescaling the image to fix the pixel aspect ratio runs the risk of making
67 // the aspect ratio worse, if QTKit selects a new source mode with a different
68 // shape. This constant ensures that we don't take this risk if the current
69 // aspect ratio is tolerable.
70 const float kMaxPixelAspectRatio = 1.15;
72 // The following constants are extracted from the specification "Universal
73 // Serial Bus Device Class Definition for Video Devices", Rev. 1.1 June 1, 2005.
74 // http://www.usb.org/developers/devclass_docs/USB_Video_Class_1_1.zip
75 // CS_INTERFACE: Sec. A.4 "Video Class-Specific Descriptor Types".
76 const int kVcCsInterface = 0x24;
77 // VC_PROCESSING_UNIT: Sec. A.5 "Video Class-Specific VC Interface Descriptor
79 const int kVcProcessingUnit = 0x5;
80 // SET_CUR: Sec. A.8 "Video Class-Specific Request Codes".
81 const int kVcRequestCodeSetCur = 0x1;
82 // PU_POWER_LINE_FREQUENCY_CONTROL: Sec. A.9.5 "Processing Unit Control
84 const int kPuPowerLineFrequencyControl = 0x5;
85 // Sec. 4.2.2.3.5 Power Line Frequency Control.
88 const int kPuPowerLineFrequencyControlCommandSize = 1;
90 // Addition to the IOUSB family of structures, with subtype and unit ID.
91 typedef struct IOUSBInterfaceDescriptor {
92 IOUSBDescriptorHeader header;
93 UInt8 bDescriptorSubType;
95 } IOUSBInterfaceDescriptor;
97 static void GetBestMatchSupportedResolution(gfx::Size* resolution) {
98 int min_diff = kint32max;
99 const int desired_area = resolution->GetArea();
100 for (size_t i = 0; i < arraysize(kWellSupportedResolutions); ++i) {
101 const int area = kWellSupportedResolutions[i]->width *
102 kWellSupportedResolutions[i]->height;
103 const int diff = std::abs(desired_area - area);
104 if (diff < min_diff) {
106 resolution->SetSize(kWellSupportedResolutions[i]->width,
107 kWellSupportedResolutions[i]->height);
112 // Tries to create a user-side device interface for a given USB device. Returns
113 // true if interface was found and passes it back in |device_interface|. The
114 // caller should release |device_interface|.
115 static bool FindDeviceInterfaceInUsbDevice(
117 const int product_id,
118 const io_service_t usb_device,
119 IOUSBDeviceInterface*** device_interface) {
120 // Create a plugin, i.e. a user-side controller to manipulate USB device.
121 IOCFPlugInInterface** plugin;
122 SInt32 score; // Unused, but required for IOCreatePlugInInterfaceForService.
123 kern_return_t kr = IOCreatePlugInInterfaceForService(
124 usb_device, kIOUSBDeviceUserClientTypeID, kIOCFPlugInInterfaceID, &plugin,
126 if (kr != kIOReturnSuccess || !plugin) {
127 DLOG(ERROR) << "IOCreatePlugInInterfaceForService";
130 base::mac::ScopedIOPluginInterface<IOCFPlugInInterface> plugin_ref(plugin);
132 // Fetch the Device Interface from the plugin.
133 HRESULT res = (*plugin)->QueryInterface(
134 plugin, CFUUIDGetUUIDBytes(kIOUSBDeviceInterfaceID),
135 reinterpret_cast<LPVOID*>(device_interface));
136 if (!SUCCEEDED(res) || !*device_interface) {
137 DLOG(ERROR) << "QueryInterface, couldn't create interface to USB";
143 // Tries to find a Video Control type interface inside a general USB device
144 // interface |device_interface|, and returns it in |video_control_interface| if
145 // found. The returned interface must be released in the caller.
146 static bool FindVideoControlInterfaceInDeviceInterface(
147 IOUSBDeviceInterface** device_interface,
148 IOCFPlugInInterface*** video_control_interface) {
149 // Create an iterator to the list of Video-AVControl interfaces of the device,
150 // then get the first interface in the list.
151 io_iterator_t interface_iterator;
152 IOUSBFindInterfaceRequest interface_request = {
153 .bInterfaceClass = kUSBVideoInterfaceClass,
154 .bInterfaceSubClass = kUSBVideoControlSubClass,
155 .bInterfaceProtocol = kIOUSBFindInterfaceDontCare,
156 .bAlternateSetting = kIOUSBFindInterfaceDontCare};
159 ->CreateInterfaceIterator(device_interface, &interface_request,
160 &interface_iterator);
161 if (kr != kIOReturnSuccess) {
162 DLOG(ERROR) << "Could not create an iterator to the device's interfaces.";
165 base::mac::ScopedIOObject<io_iterator_t> iterator_ref(interface_iterator);
167 // There should be just one interface matching the class-subclass desired.
168 io_service_t found_interface;
169 found_interface = IOIteratorNext(interface_iterator);
170 if (!found_interface) {
171 DLOG(ERROR) << "Could not find a Video-AVControl interface in the device.";
174 base::mac::ScopedIOObject<io_service_t> found_interface_ref(found_interface);
176 // Create a user side controller (i.e. a "plugin") for the found interface.
178 kr = IOCreatePlugInInterfaceForService(
179 found_interface, kIOUSBInterfaceUserClientTypeID, kIOCFPlugInInterfaceID,
180 video_control_interface, &score);
181 if (kr != kIOReturnSuccess || !*video_control_interface) {
182 DLOG(ERROR) << "IOCreatePlugInInterfaceForService";
188 // Creates a control interface for |plugin_interface| and produces a command to
189 // set the appropriate Power Line frequency for flicker removal.
190 static void SetAntiFlickerInVideoControlInterface(
191 IOCFPlugInInterface** plugin_interface,
192 const int frequency) {
193 // Create, the control interface for the found plugin, and release
194 // the intermediate plugin.
195 IOUSBInterfaceInterface** control_interface = NULL;
198 ->QueryInterface(plugin_interface,
199 CFUUIDGetUUIDBytes(kIOUSBInterfaceInterfaceID),
200 reinterpret_cast<LPVOID*>(&control_interface));
201 if (!SUCCEEDED(res) || !control_interface) {
202 DLOG(ERROR) << "Couldn’t create control interface";
205 base::mac::ScopedIOPluginInterface<IOUSBInterfaceInterface>
206 control_interface_ref(control_interface);
208 // Find the device's unit ID presenting type 0x24 (kVcCsInterface) and
209 // subtype 0x5 (kVcProcessingUnit). Inside this unit is where we find the
210 // power line frequency removal setting, and this id is device dependent.
211 int real_unit_id = -1;
212 IOUSBDescriptorHeader* descriptor = NULL;
213 IOUSBInterfaceDescriptor* cs_descriptor = NULL;
214 IOUSBInterfaceInterface220** interface =
215 reinterpret_cast<IOUSBInterfaceInterface220**>(control_interface);
216 while ((descriptor = (*interface)
217 ->FindNextAssociatedDescriptor(interface, descriptor,
219 cs_descriptor = reinterpret_cast<IOUSBInterfaceDescriptor*>(descriptor);
220 if ((descriptor->bDescriptorType == kVcCsInterface) &&
221 (cs_descriptor->bDescriptorSubType == kVcProcessingUnit)) {
222 real_unit_id = cs_descriptor->bUnitID;
226 DVLOG_IF(1, real_unit_id == -1)
227 << "This USB device doesn't seem to have a "
228 << " VC_PROCESSING_UNIT, anti-flicker not available";
229 if (real_unit_id == -1)
232 if ((*control_interface)->USBInterfaceOpen(control_interface) !=
234 DLOG(ERROR) << "Unable to open control interface";
238 // Create the control request and launch it to the device's control interface.
239 // Note how the wIndex needs the interface number OR'ed in the lowest bits.
240 IOUSBDevRequest command;
241 command.bmRequestType =
242 USBmakebmRequestType(kUSBOut, kUSBClass, kUSBInterface);
243 command.bRequest = kVcRequestCodeSetCur;
244 UInt8 interface_number;
246 ->GetInterfaceNumber(control_interface, &interface_number);
247 command.wIndex = (real_unit_id << 8) | interface_number;
248 const int selector = kPuPowerLineFrequencyControl;
249 command.wValue = (selector << 8);
250 command.wLength = kPuPowerLineFrequencyControlCommandSize;
251 command.wLenDone = 0;
252 int power_line_flag_value = (frequency == 50) ? k50Hz : k60Hz;
253 command.pData = &power_line_flag_value;
256 (*control_interface)->ControlRequest(control_interface, 0, &command);
257 DLOG_IF(ERROR, ret != kIOReturnSuccess) << "Anti-flicker control request"
258 << " failed (0x" << std::hex << ret
259 << "), unit id: " << real_unit_id;
260 DVLOG_IF(1, ret == kIOReturnSuccess) << "Anti-flicker set to " << frequency
263 (*control_interface)->USBInterfaceClose(control_interface);
266 // Sets the flicker removal in a USB webcam identified by |vendor_id| and
267 // |product_id|, if available. The process includes first finding all USB
268 // devices matching the specified |vendor_id| and |product_id|; for each
269 // matching device, a device interface, and inside it a video control interface
270 // are created. The latter is used to a send a power frequency setting command.
271 static void SetAntiFlickerInUsbDevice(const int vendor_id,
272 const int product_id,
273 const int frequency) {
276 DVLOG(1) << "Setting Power Line Frequency to " << frequency << " Hz, device "
277 << std::hex << vendor_id << "-" << product_id;
279 // Compose a search dictionary with vendor and product ID.
280 CFMutableDictionaryRef query_dictionary =
281 IOServiceMatching(kIOUSBDeviceClassName);
282 CFDictionarySetValue(
283 query_dictionary, CFSTR(kUSBVendorName),
284 CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &vendor_id));
285 CFDictionarySetValue(
286 query_dictionary, CFSTR(kUSBProductName),
287 CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &product_id));
289 io_iterator_t usb_iterator;
290 kern_return_t kr = IOServiceGetMatchingServices(
291 kIOMasterPortDefault, query_dictionary, &usb_iterator);
292 if (kr != kIOReturnSuccess) {
293 DLOG(ERROR) << "No devices found with specified Vendor and Product ID.";
296 base::mac::ScopedIOObject<io_iterator_t> usb_iterator_ref(usb_iterator);
298 while (io_service_t usb_device = IOIteratorNext(usb_iterator)) {
299 base::mac::ScopedIOObject<io_service_t> usb_device_ref(usb_device);
301 IOUSBDeviceInterface** device_interface = NULL;
302 if (!FindDeviceInterfaceInUsbDevice(vendor_id, product_id, usb_device,
303 &device_interface)) {
306 base::mac::ScopedIOPluginInterface<IOUSBDeviceInterface>
307 device_interface_ref(device_interface);
309 IOCFPlugInInterface** video_control_interface = NULL;
310 if (!FindVideoControlInterfaceInDeviceInterface(device_interface,
311 &video_control_interface)) {
314 base::mac::ScopedIOPluginInterface<IOCFPlugInInterface>
315 plugin_interface_ref(video_control_interface);
317 SetAntiFlickerInVideoControlInterface(video_control_interface, frequency);
321 const std::string VideoCaptureDevice::Name::GetModel() const {
322 // Skip the AVFoundation's not USB nor built-in devices.
323 if (capture_api_type() == AVFOUNDATION && transport_type() != USB_OR_BUILT_IN)
325 if (capture_api_type() == DECKLINK)
327 // Both PID and VID are 4 characters.
328 if (unique_id_.size() < 2 * kVidPidSize)
331 // The last characters of device id is a concatenation of VID and then PID.
332 const size_t vid_location = unique_id_.size() - 2 * kVidPidSize;
333 std::string id_vendor = unique_id_.substr(vid_location, kVidPidSize);
334 const size_t pid_location = unique_id_.size() - kVidPidSize;
335 std::string id_product = unique_id_.substr(pid_location, kVidPidSize);
337 return id_vendor + ":" + id_product;
340 VideoCaptureDeviceMac::VideoCaptureDeviceMac(const Name& device_name)
341 : device_name_(device_name),
342 tried_to_square_pixels_(false),
343 task_runner_(base::ThreadTaskRunnerHandle::Get()),
344 state_(kNotInitialized),
345 capture_device_(nil),
346 weak_factory_(this) {
347 // Avoid reconfiguring AVFoundation or blacklisted devices.
348 final_resolution_selected_ = AVFoundationGlue::IsAVFoundationSupported() ||
349 device_name.is_blacklisted();
352 VideoCaptureDeviceMac::~VideoCaptureDeviceMac() {
353 DCHECK(task_runner_->BelongsToCurrentThread());
354 [capture_device_ release];
357 void VideoCaptureDeviceMac::AllocateAndStart(
358 const VideoCaptureParams& params,
359 scoped_ptr<VideoCaptureDevice::Client> client) {
360 DCHECK(task_runner_->BelongsToCurrentThread());
361 if (state_ != kIdle) {
365 // QTKit API can scale captured frame to any size requested, which would lead
366 // to undesired aspect ratio changes. Try to open the camera with a known
367 // supported format and let the client crop/pad the captured frames.
368 gfx::Size resolution = params.requested_format.frame_size;
369 if (!AVFoundationGlue::IsAVFoundationSupported())
370 GetBestMatchSupportedResolution(&resolution);
372 client_ = client.Pass();
373 if (device_name_.capture_api_type() == Name::AVFOUNDATION)
374 LogMessage("Using AVFoundation for device: " + device_name_.name());
376 LogMessage("Using QTKit for device: " + device_name_.name());
378 [NSString stringWithUTF8String:device_name_.id().c_str()];
380 [capture_device_ setFrameReceiver:this];
382 if (![capture_device_ setCaptureDevice:deviceId]) {
383 SetErrorState("Could not open capture device.");
387 capture_format_.frame_size = resolution;
388 capture_format_.frame_rate =
389 std::max(kMinFrameRate,
390 std::min(params.requested_format.frame_rate, kMaxFrameRate));
391 // Leave the pixel format selection to AVFoundation/QTKit. The pixel format
392 // will be passed to |ReceiveFrame|.
393 capture_format_.pixel_format = VIDEO_CAPTURE_PIXEL_FORMAT_UNKNOWN;
395 // QTKit: Set the capture resolution only if this is VGA or smaller, otherwise
396 // leave it unconfigured and start capturing: QTKit will produce frames at the
397 // native resolution, allowing us to identify cameras whose native resolution
398 // is too low for HD. This additional information comes at a cost in startup
399 // latency, because the webcam will need to be reopened if its default
400 // resolution is not HD or VGA.
401 // AVfoundation is configured for all resolutions.
402 if (AVFoundationGlue::IsAVFoundationSupported() ||
403 resolution.width() <= kVGA.width || resolution.height() <= kVGA.height) {
404 if (!UpdateCaptureResolution())
408 // Try setting the power line frequency removal (anti-flicker). The built-in
409 // cameras are normally suspended so the configuration must happen right
410 // before starting capture and during configuration.
411 const std::string& device_model = device_name_.GetModel();
412 if (device_model.length() > 2 * kVidPidSize) {
413 std::string vendor_id = device_model.substr(0, kVidPidSize);
414 std::string model_id = device_model.substr(kVidPidSize + 1);
415 int vendor_id_as_int, model_id_as_int;
416 if (base::HexStringToInt(base::StringPiece(vendor_id), &vendor_id_as_int) &&
417 base::HexStringToInt(base::StringPiece(model_id), &model_id_as_int)) {
418 SetAntiFlickerInUsbDevice(vendor_id_as_int, model_id_as_int,
419 GetPowerLineFrequencyForLocation());
423 if (![capture_device_ startCapture]) {
424 SetErrorState("Could not start capture device.");
431 void VideoCaptureDeviceMac::StopAndDeAllocate() {
432 DCHECK(task_runner_->BelongsToCurrentThread());
433 DCHECK(state_ == kCapturing || state_ == kError) << state_;
435 [capture_device_ setCaptureDevice:nil];
436 [capture_device_ setFrameReceiver:nil];
439 tried_to_square_pixels_ = false;
442 bool VideoCaptureDeviceMac::Init(
443 VideoCaptureDevice::Name::CaptureApiType capture_api_type) {
444 DCHECK(task_runner_->BelongsToCurrentThread());
445 DCHECK_EQ(state_, kNotInitialized);
447 if (capture_api_type == Name::AVFOUNDATION) {
449 [[VideoCaptureDeviceAVFoundation alloc] initWithFrameReceiver:this];
452 [[VideoCaptureDeviceQTKit alloc] initWithFrameReceiver:this];
455 if (!capture_device_)
462 void VideoCaptureDeviceMac::ReceiveFrame(const uint8* video_frame,
463 int video_frame_length,
464 const VideoCaptureFormat& frame_format,
465 int aspect_numerator,
466 int aspect_denominator) {
467 // This method is safe to call from a device capture thread, i.e. any thread
468 // controlled by QTKit/AVFoundation.
469 if (!final_resolution_selected_) {
470 DCHECK(!AVFoundationGlue::IsAVFoundationSupported());
471 if (capture_format_.frame_size.width() > kVGA.width ||
472 capture_format_.frame_size.height() > kVGA.height) {
473 // We are requesting HD. Make sure that the picture is good, otherwise
475 bool change_to_vga = false;
476 if (frame_format.frame_size.width() <
477 capture_format_.frame_size.width() ||
478 frame_format.frame_size.height() <
479 capture_format_.frame_size.height()) {
480 // These are the default capture settings, not yet configured to match
481 // |capture_format_|.
482 DCHECK(frame_format.frame_rate == 0);
483 DVLOG(1) << "Switching to VGA because the default resolution is "
484 << frame_format.frame_size.ToString();
485 change_to_vga = true;
488 if (capture_format_.frame_size == frame_format.frame_size &&
489 aspect_numerator != aspect_denominator) {
490 DVLOG(1) << "Switching to VGA because HD has nonsquare pixel "
491 << "aspect ratio " << aspect_numerator << ":"
492 << aspect_denominator;
493 change_to_vga = true;
497 capture_format_.frame_size.SetSize(kVGA.width, kVGA.height);
500 if (capture_format_.frame_size == frame_format.frame_size &&
501 !tried_to_square_pixels_ &&
502 (aspect_numerator > kMaxPixelAspectRatio * aspect_denominator ||
503 aspect_denominator > kMaxPixelAspectRatio * aspect_numerator)) {
504 // The requested size results in non-square PAR. Shrink the frame to 1:1
505 // PAR (assuming QTKit selects the same input mode, which is not
507 int new_width = capture_format_.frame_size.width();
508 int new_height = capture_format_.frame_size.height();
509 if (aspect_numerator < aspect_denominator)
510 new_width = (new_width * aspect_numerator) / aspect_denominator;
512 new_height = (new_height * aspect_denominator) / aspect_numerator;
513 capture_format_.frame_size.SetSize(new_width, new_height);
514 tried_to_square_pixels_ = true;
517 if (capture_format_.frame_size == frame_format.frame_size) {
518 final_resolution_selected_ = true;
520 UpdateCaptureResolution();
521 // Let the resolution update sink through QTKit and wait for next frame.
526 // QTKit capture source can change resolution if someone else reconfigures the
527 // camera, and that is fine: http://crbug.com/353620. In AVFoundation, this
528 // should not happen, it should resize internally.
529 if (!AVFoundationGlue::IsAVFoundationSupported()) {
530 capture_format_.frame_size = frame_format.frame_size;
531 } else if (capture_format_.frame_size != frame_format.frame_size) {
532 ReceiveError("Captured resolution " + frame_format.frame_size.ToString() +
533 ", and expected " + capture_format_.frame_size.ToString());
537 client_->OnIncomingCapturedData(video_frame, video_frame_length, frame_format,
538 0, base::TimeTicks::Now());
541 void VideoCaptureDeviceMac::ReceiveError(const std::string& reason) {
542 task_runner_->PostTask(FROM_HERE,
543 base::Bind(&VideoCaptureDeviceMac::SetErrorState,
544 weak_factory_.GetWeakPtr(), reason));
547 void VideoCaptureDeviceMac::SetErrorState(const std::string& reason) {
548 DCHECK(task_runner_->BelongsToCurrentThread());
550 client_->OnError(reason);
553 void VideoCaptureDeviceMac::LogMessage(const std::string& message) {
554 DCHECK(task_runner_->BelongsToCurrentThread());
556 client_->OnLog(message);
559 bool VideoCaptureDeviceMac::UpdateCaptureResolution() {
560 if (![capture_device_ setCaptureHeight:capture_format_.frame_size.height()
561 width:capture_format_.frame_size.width()
562 frameRate:capture_format_.frame_rate]) {
563 ReceiveError("Could not configure capture device.");