content/common/gpu/media/vt_video_decode_accelerator.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <algorithm>
   6
   7 #include <CoreVideo/CoreVideo.h>
   8 #include <OpenGL/CGLIOSurface.h>
   9 #include <OpenGL/gl.h>
  10
  11 #include "base/bind.h"
  12 #include "base/command_line.h"
  13 #include "base/logging.h"
  14 #include "base/mac/mac_logging.h"
  15 #include "base/metrics/histogram_macros.h"
  16 #include "base/sys_byteorder.h"
  17 #include "base/thread_task_runner_handle.h"
  18 #include "content/common/gpu/media/vt_video_decode_accelerator.h"
  19 #include "content/public/common/content_switches.h"
  20 #include "media/base/limits.h"
  21 #include "ui/gl/scoped_binders.h"
  22
  23 using content_common_gpu_media::kModuleVt;
  24 using content_common_gpu_media::InitializeStubs;
  25 using content_common_gpu_media::IsVtInitialized;
  26 using content_common_gpu_media::StubPathMap;
  27
  28 #define NOTIFY_STATUS(name, status, session_failure)   \
  29     do {                                               \
  30       OSSTATUS_DLOG(ERROR, status) << name;            \
  31       NotifyError(PLATFORM_FAILURE, session_failure);  \
  32     } while (0)
  33
  34 namespace content {
  35
  36 // Size to use for NALU length headers in AVC format (can be 1, 2, or 4).
  37 static const int kNALUHeaderLength = 4;
  38
  39 // We request 5 picture buffers from the client, each of which has a texture ID
  40 // that we can bind decoded frames to. We need enough to satisfy preroll, and
  41 // enough to avoid unnecessary stalling, but no more than that. The resource
  42 // requirements are low, as we don't need the textures to be backed by storage.
  43 static const int kNumPictureBuffers = media::limits::kMaxVideoFrames + 1;
  44
  45 // Maximum number of frames to queue for reordering before we stop asking for
  46 // more. (NotifyEndOfBitstreamBuffer() is called when frames are moved into the
  47 // reorder queue.)
  48 static const int kMaxReorderQueueSize = 16;
  49
  50 // Logged to UMA, so never reuse values. Make sure to update
  51 // VTVDAInitializationFailureType in histograms.xml to match.
  52 enum VTVDAInitializationFailureType {
  53   IFT_SUCCESSFULLY_INITIALIZED = 0,
  54   IFT_FRAMEWORK_LOAD_ERROR = 1,
  55   IFT_HARDWARE_SESSION_ERROR = 2,
  56   IFT_SOFTWARE_SESSION_ERROR = 3,
  57   // Must always be equal to largest entry logged.
  58   IFT_MAX = IFT_SOFTWARE_SESSION_ERROR
  59 };
  60
  61 static void ReportInitializationFailure(
  62     VTVDAInitializationFailureType failure_type) {
  63   DCHECK_LT(failure_type, IFT_MAX + 1);
  64   UMA_HISTOGRAM_ENUMERATION("Media.VTVDA.InitializationFailureReason",
  65                             failure_type,
  66                             IFT_MAX + 1);
  67 }
  68
  69 // Build an |image_config| dictionary for VideoToolbox initialization.
  70 static base::ScopedCFTypeRef<CFMutableDictionaryRef>
  71 BuildImageConfig(CMVideoDimensions coded_dimensions) {
  72   base::ScopedCFTypeRef<CFMutableDictionaryRef> image_config;
  73
  74   // TODO(sandersd): Does it save some work or memory to use 4:2:0?
  75   int32_t pixel_format = kCVPixelFormatType_422YpCbCr8;
  76 #define CFINT(i) CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &i)
  77   base::ScopedCFTypeRef<CFNumberRef> cf_pixel_format(CFINT(pixel_format));
  78   base::ScopedCFTypeRef<CFNumberRef> cf_width(CFINT(coded_dimensions.width));
  79   base::ScopedCFTypeRef<CFNumberRef> cf_height(CFINT(coded_dimensions.height));
  80 #undef CFINT
  81   if (!cf_pixel_format.get() || !cf_width.get() || !cf_height.get())
  82     return image_config;
  83
  84   image_config.reset(
  85       CFDictionaryCreateMutable(
  86           kCFAllocatorDefault,
  87           4,  // capacity
  88           &kCFTypeDictionaryKeyCallBacks,
  89           &kCFTypeDictionaryValueCallBacks));
  90   if (!image_config.get())
  91     return image_config;
  92
  93   CFDictionarySetValue(image_config, kCVPixelBufferPixelFormatTypeKey,
  94                        cf_pixel_format);
  95   CFDictionarySetValue(image_config, kCVPixelBufferWidthKey, cf_width);
  96   CFDictionarySetValue(image_config, kCVPixelBufferHeightKey, cf_height);
  97   CFDictionarySetValue(image_config, kCVPixelBufferOpenGLCompatibilityKey,
  98                        kCFBooleanTrue);
  99
 100   return image_config;
 101 }
 102
 103 // Create a VTDecompressionSession using the provided |pps| and |sps|. If
 104 // |require_hardware| is true, the session must uses real hardware decoding
 105 // (as opposed to software decoding inside of VideoToolbox) to be considered
 106 // successful.
 107 //
 108 // TODO(sandersd): Merge with ConfigureDecoder(), as the code is very similar.
 109 static bool CreateVideoToolboxSession(const uint8_t* sps, size_t sps_size,
 110                                       const uint8_t* pps, size_t pps_size,
 111                                       bool require_hardware) {
 112   const uint8_t* data_ptrs[] = {sps, pps};
 113   const size_t data_sizes[] = {sps_size, pps_size};
 114
 115   base::ScopedCFTypeRef<CMFormatDescriptionRef> format;
 116   OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(
 117       kCFAllocatorDefault,
 118       2,                          // parameter_set_count
 119       data_ptrs,                  // &parameter_set_pointers
 120       data_sizes,                 // &parameter_set_sizes
 121       kNALUHeaderLength,          // nal_unit_header_length
 122       format.InitializeInto());
 123   if (status) {
 124     OSSTATUS_LOG(ERROR, status) << "Failed to create CMVideoFormatDescription.";
 125     return false;
 126   }
 127
 128   base::ScopedCFTypeRef<CFMutableDictionaryRef> decoder_config(
 129       CFDictionaryCreateMutable(
 130           kCFAllocatorDefault,
 131           1,  // capacity
 132           &kCFTypeDictionaryKeyCallBacks,
 133           &kCFTypeDictionaryValueCallBacks));
 134   if (!decoder_config.get())
 135     return false;
 136
 137   if (require_hardware) {
 138     CFDictionarySetValue(
 139         decoder_config,
 140         // kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
 141         CFSTR("RequireHardwareAcceleratedVideoDecoder"),
 142         kCFBooleanTrue);
 143   }
 144
 145   base::ScopedCFTypeRef<CFMutableDictionaryRef> image_config(
 146       BuildImageConfig(CMVideoFormatDescriptionGetDimensions(format)));
 147   if (!image_config.get())
 148     return false;
 149
 150   VTDecompressionOutputCallbackRecord callback = {0};
 151
 152   base::ScopedCFTypeRef<VTDecompressionSessionRef> session;
 153   status = VTDecompressionSessionCreate(
 154       kCFAllocatorDefault,
 155       format,               // video_format_description
 156       decoder_config,       // video_decoder_specification
 157       image_config,         // destination_image_buffer_attributes
 158       &callback,            // output_callback
 159       session.InitializeInto());
 160   if (status) {
 161     ReportInitializationFailure(require_hardware ? IFT_HARDWARE_SESSION_ERROR
 162                                                  : IFT_SOFTWARE_SESSION_ERROR);
 163     OSSTATUS_LOG(ERROR, status) << "Failed to create VTDecompressionSession";
 164     return false;
 165   }
 166
 167   return true;
 168 }
 169
 170 // The purpose of this function is to preload the generic and hardware-specific
 171 // libraries required by VideoToolbox before the GPU sandbox is enabled.
 172 // VideoToolbox normally loads the hardware-specific libraries lazily, so we
 173 // must actually create a decompression session. If creating a decompression
 174 // session fails, hardware decoding will be disabled (Initialize() will always
 175 // return false).
 176 static bool InitializeVideoToolboxInternal() {
 177   if (base::CommandLine::ForCurrentProcess()->HasSwitch(
 178           switches::kDisableAcceleratedVideoDecode)) {
 179     return false;
 180   }
 181
 182   if (!IsVtInitialized()) {
 183     // CoreVideo is also required, but the loader stops after the first path is
 184     // loaded. Instead we rely on the transitive dependency from VideoToolbox to
 185     // CoreVideo.
 186     // TODO(sandersd): Fallback to PrivateFrameworks to support OS X < 10.8.
 187     StubPathMap paths;
 188     paths[kModuleVt].push_back(FILE_PATH_LITERAL(
 189         "/System/Library/Frameworks/VideoToolbox.framework/VideoToolbox"));
 190     if (!InitializeStubs(paths)) {
 191       ReportInitializationFailure(IFT_FRAMEWORK_LOAD_ERROR);
 192       LOG(ERROR) << "Failed to initialize VideoToobox framework. "
 193                  << "Hardware accelerated video decoding will be disabled.";
 194       return false;
 195     }
 196   }
 197
 198   // Create a hardware decoding session.
 199   // SPS and PPS data are taken from a 480p sample (buck2.mp4).
 200   const uint8_t sps_normal[] = {0x67, 0x64, 0x00, 0x1e, 0xac, 0xd9, 0x80, 0xd4,
 201                                 0x3d, 0xa1, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00,
 202                                 0x00, 0x03, 0x00, 0x30, 0x8f, 0x16, 0x2d, 0x9a};
 203   const uint8_t pps_normal[] = {0x68, 0xe9, 0x7b, 0xcb};
 204   if (!CreateVideoToolboxSession(sps_normal, arraysize(sps_normal), pps_normal,
 205                                  arraysize(pps_normal), true)) {
 206     LOG(ERROR) << "Failed to create hardware VideoToolbox session. "
 207                << "Hardware accelerated video decoding will be disabled.";
 208     return false;
 209   }
 210
 211   // Create a software decoding session.
 212   // SPS and PPS data are taken from a 18p sample (small2.mp4).
 213   const uint8_t sps_small[] = {0x67, 0x64, 0x00, 0x0a, 0xac, 0xd9, 0x89, 0x7e,
 214                                0x22, 0x10, 0x00, 0x00, 0x3e, 0x90, 0x00, 0x0e,
 215                                0xa6, 0x08, 0xf1, 0x22, 0x59, 0xa0};
 216   const uint8_t pps_small[] = {0x68, 0xe9, 0x79, 0x72, 0xc0};
 217   if (!CreateVideoToolboxSession(sps_small, arraysize(sps_small), pps_small,
 218                                  arraysize(pps_small), false)) {
 219     LOG(ERROR) << "Failed to create software VideoToolbox session. "
 220                << "Hardware accelerated video decoding will be disabled.";
 221     return false;
 222   }
 223
 224   ReportInitializationFailure(IFT_SUCCESSFULLY_INITIALIZED);
 225   return true;
 226 }
 227
 228 bool InitializeVideoToolbox() {
 229   // InitializeVideoToolbox() is called only from the GPU process main thread;
 230   // once for sandbox warmup, and then once each time a VTVideoDecodeAccelerator
 231   // is initialized.
 232   static bool attempted = false;
 233   static bool succeeded = false;
 234
 235   if (!attempted) {
 236     attempted = true;
 237     succeeded = InitializeVideoToolboxInternal();
 238   }
 239
 240   return succeeded;
 241 }
 242
 243 // Route decoded frame callbacks back into the VTVideoDecodeAccelerator.
 244 static void OutputThunk(
 245     void* decompression_output_refcon,
 246     void* source_frame_refcon,
 247     OSStatus status,
 248     VTDecodeInfoFlags info_flags,
 249     CVImageBufferRef image_buffer,
 250     CMTime presentation_time_stamp,
 251     CMTime presentation_duration) {
 252   VTVideoDecodeAccelerator* vda =
 253       reinterpret_cast<VTVideoDecodeAccelerator*>(decompression_output_refcon);
 254   vda->Output(source_frame_refcon, status, image_buffer);
 255 }
 256
 257 VTVideoDecodeAccelerator::Task::Task(TaskType type) : type(type) {
 258 }
 259
 260 VTVideoDecodeAccelerator::Task::~Task() {
 261 }
 262
 263 VTVideoDecodeAccelerator::Frame::Frame(int32_t bitstream_id)
 264     : bitstream_id(bitstream_id), pic_order_cnt(0), reorder_window(0) {
 265 }
 266
 267 VTVideoDecodeAccelerator::Frame::~Frame() {
 268 }
 269
 270 bool VTVideoDecodeAccelerator::FrameOrder::operator()(
 271     const linked_ptr<Frame>& lhs,
 272     const linked_ptr<Frame>& rhs) const {
 273   if (lhs->pic_order_cnt != rhs->pic_order_cnt)
 274     return lhs->pic_order_cnt > rhs->pic_order_cnt;
 275   // If |pic_order_cnt| is the same, fall back on using the bitstream order.
 276   // TODO(sandersd): Assign a sequence number in Decode() and use that instead.
 277   // TODO(sandersd): Using the sequence number, ensure that frames older than
 278   // |kMaxReorderQueueSize| are ordered first, regardless of |pic_order_cnt|.
 279   return lhs->bitstream_id > rhs->bitstream_id;
 280 }
 281
 282 VTVideoDecodeAccelerator::VTVideoDecodeAccelerator(
 283     CGLContextObj cgl_context,
 284     const base::Callback<bool(void)>& make_context_current)
 285     : cgl_context_(cgl_context),
 286       make_context_current_(make_context_current),
 287       client_(nullptr),
 288       state_(STATE_DECODING),
 289       format_(nullptr),
 290       session_(nullptr),
 291       last_sps_id_(-1),
 292       last_pps_id_(-1),
 293       gpu_task_runner_(base::ThreadTaskRunnerHandle::Get()),
 294       decoder_thread_("VTDecoderThread"),
 295       weak_this_factory_(this) {
 296   DCHECK(!make_context_current_.is_null());
 297   callback_.decompressionOutputCallback = OutputThunk;
 298   callback_.decompressionOutputRefCon = this;
 299   weak_this_ = weak_this_factory_.GetWeakPtr();
 300 }
 301
 302 VTVideoDecodeAccelerator::~VTVideoDecodeAccelerator() {
 303 }
 304
 305 bool VTVideoDecodeAccelerator::Initialize(
 306     media::VideoCodecProfile profile,
 307     Client* client) {
 308   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 309   client_ = client;
 310
 311   if (!InitializeVideoToolbox())
 312     return false;
 313
 314   // Only H.264 with 4:2:0 chroma sampling is supported.
 315   if (profile < media::H264PROFILE_MIN ||
 316       profile > media::H264PROFILE_MAX ||
 317       profile == media::H264PROFILE_HIGH422PROFILE ||
 318       profile == media::H264PROFILE_HIGH444PREDICTIVEPROFILE) {
 319     return false;
 320   }
 321
 322   // Spawn a thread to handle parsing and calling VideoToolbox.
 323   if (!decoder_thread_.Start())
 324     return false;
 325
 326   // Count the session as successfully initialized.
 327   UMA_HISTOGRAM_ENUMERATION("Media.VTVDA.SessionFailureReason",
 328                             SFT_SUCCESSFULLY_INITIALIZED,
 329                             SFT_MAX + 1);
 330   return true;
 331 }
 332
 333 bool VTVideoDecodeAccelerator::FinishDelayedFrames() {
 334   DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
 335   if (session_) {
 336     OSStatus status = VTDecompressionSessionWaitForAsynchronousFrames(session_);
 337     if (status) {
 338       NOTIFY_STATUS("VTDecompressionSessionWaitForAsynchronousFrames()",
 339                     status, SFT_PLATFORM_ERROR);
 340       return false;
 341     }
 342   }
 343   return true;
 344 }
 345
 346 bool VTVideoDecodeAccelerator::ConfigureDecoder() {
 347   DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
 348   DCHECK(!last_sps_.empty());
 349   DCHECK(!last_pps_.empty());
 350
 351   // Build the configuration records.
 352   std::vector<const uint8_t*> nalu_data_ptrs;
 353   std::vector<size_t> nalu_data_sizes;
 354   nalu_data_ptrs.reserve(3);
 355   nalu_data_sizes.reserve(3);
 356   nalu_data_ptrs.push_back(&last_sps_.front());
 357   nalu_data_sizes.push_back(last_sps_.size());
 358   if (!last_spsext_.empty()) {
 359     nalu_data_ptrs.push_back(&last_spsext_.front());
 360     nalu_data_sizes.push_back(last_spsext_.size());
 361   }
 362   nalu_data_ptrs.push_back(&last_pps_.front());
 363   nalu_data_sizes.push_back(last_pps_.size());
 364
 365   // Construct a new format description from the parameter sets.
 366   // TODO(sandersd): Replace this with custom code to support OS X < 10.9.
 367   format_.reset();
 368   OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(
 369       kCFAllocatorDefault,
 370       nalu_data_ptrs.size(),      // parameter_set_count
 371       &nalu_data_ptrs.front(),    // &parameter_set_pointers
 372       &nalu_data_sizes.front(),   // &parameter_set_sizes
 373       kNALUHeaderLength,          // nal_unit_header_length
 374       format_.InitializeInto());
 375   if (status) {
 376     NOTIFY_STATUS("CMVideoFormatDescriptionCreateFromH264ParameterSets()",
 377                   status, SFT_PLATFORM_ERROR);
 378     return false;
 379   }
 380
 381   // Store the new configuration data.
 382   CMVideoDimensions coded_dimensions =
 383       CMVideoFormatDescriptionGetDimensions(format_);
 384   coded_size_.SetSize(coded_dimensions.width, coded_dimensions.height);
 385
 386   // If the session is compatible, there's nothing else to do.
 387   if (session_ &&
 388       VTDecompressionSessionCanAcceptFormatDescription(session_, format_)) {
 389     return true;
 390   }
 391
 392   // Prepare VideoToolbox configuration dictionaries.
 393   base::ScopedCFTypeRef<CFMutableDictionaryRef> decoder_config(
 394       CFDictionaryCreateMutable(
 395           kCFAllocatorDefault,
 396           1,  // capacity
 397           &kCFTypeDictionaryKeyCallBacks,
 398           &kCFTypeDictionaryValueCallBacks));
 399   if (!decoder_config.get()) {
 400     DLOG(ERROR) << "Failed to create CFMutableDictionary.";
 401     NotifyError(PLATFORM_FAILURE, SFT_PLATFORM_ERROR);
 402     return false;
 403   }
 404
 405   CFDictionarySetValue(
 406       decoder_config,
 407       // kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder
 408       CFSTR("EnableHardwareAcceleratedVideoDecoder"),
 409       kCFBooleanTrue);
 410
 411   base::ScopedCFTypeRef<CFMutableDictionaryRef> image_config(
 412       BuildImageConfig(coded_dimensions));
 413   if (!image_config.get()) {
 414     DLOG(ERROR) << "Failed to create decoder image configuration.";
 415     NotifyError(PLATFORM_FAILURE, SFT_PLATFORM_ERROR);
 416     return false;
 417   }
 418
 419   // Ensure that the old decoder emits all frames before the new decoder can
 420   // emit any.
 421   if (!FinishDelayedFrames())
 422     return false;
 423
 424   session_.reset();
 425   status = VTDecompressionSessionCreate(
 426       kCFAllocatorDefault,
 427       format_,              // video_format_description
 428       decoder_config,       // video_decoder_specification
 429       image_config,         // destination_image_buffer_attributes
 430       &callback_,           // output_callback
 431       session_.InitializeInto());
 432   if (status) {
 433     NOTIFY_STATUS("VTDecompressionSessionCreate()", status,
 434                   SFT_UNSUPPORTED_STREAM_PARAMETERS);
 435     return false;
 436   }
 437
 438   // Report whether hardware decode is being used.
 439   bool using_hardware = false;
 440   base::ScopedCFTypeRef<CFBooleanRef> cf_using_hardware;
 441   if (VTSessionCopyProperty(
 442           session_,
 443           // kVTDecompressionPropertyKey_UsingHardwareAcceleratedVideoDecoder
 444           CFSTR("UsingHardwareAcceleratedVideoDecoder"),
 445           kCFAllocatorDefault,
 446           cf_using_hardware.InitializeInto()) == 0) {
 447     using_hardware = CFBooleanGetValue(cf_using_hardware);
 448   }
 449   UMA_HISTOGRAM_BOOLEAN("Media.VTVDA.HardwareAccelerated", using_hardware);
 450
 451   return true;
 452 }
 453
 454 void VTVideoDecodeAccelerator::DecodeTask(
 455     const media::BitstreamBuffer& bitstream,
 456     Frame* frame) {
 457   DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
 458
 459   // Map the bitstream buffer.
 460   base::SharedMemory memory(bitstream.handle(), true);
 461   size_t size = bitstream.size();
 462   if (!memory.Map(size)) {
 463     DLOG(ERROR) << "Failed to map bitstream buffer";
 464     NotifyError(PLATFORM_FAILURE, SFT_PLATFORM_ERROR);
 465     return;
 466   }
 467   const uint8_t* buf = static_cast<uint8_t*>(memory.memory());
 468
 469   // NALUs are stored with Annex B format in the bitstream buffer (start codes),
 470   // but VideoToolbox expects AVC format (length headers), so we must rewrite
 471   // the data.
 472   //
 473   // Locate relevant NALUs and compute the size of the rewritten data. Also
 474   // record any parameter sets for VideoToolbox initialization.
 475   bool config_changed = false;
 476   bool has_slice = false;
 477   size_t data_size = 0;
 478   std::vector<media::H264NALU> nalus;
 479   parser_.SetStream(buf, size);
 480   media::H264NALU nalu;
 481   while (true) {
 482     media::H264Parser::Result result = parser_.AdvanceToNextNALU(&nalu);
 483     if (result == media::H264Parser::kEOStream)
 484       break;
 485     if (result == media::H264Parser::kUnsupportedStream) {
 486       DLOG(ERROR) << "Unsupported H.264 stream";
 487       NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
 488       return;
 489     }
 490     if (result != media::H264Parser::kOk) {
 491       DLOG(ERROR) << "Failed to parse H.264 stream";
 492       NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
 493       return;
 494     }
 495     switch (nalu.nal_unit_type) {
 496       case media::H264NALU::kSPS:
 497         last_sps_.assign(nalu.data, nalu.data + nalu.size);
 498         last_spsext_.clear();
 499         config_changed = true;
 500         result = parser_.ParseSPS(&last_sps_id_);
 501         if (result == media::H264Parser::kUnsupportedStream) {
 502           DLOG(ERROR) << "Unsupported SPS";
 503           NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
 504           return;
 505         }
 506         if (result != media::H264Parser::kOk) {
 507           DLOG(ERROR) << "Could not parse SPS";
 508           NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
 509           return;
 510         }
 511         break;
 512
 513       case media::H264NALU::kSPSExt:
 514         // TODO(sandersd): Check that the previous NALU was an SPS.
 515         last_spsext_.assign(nalu.data, nalu.data + nalu.size);
 516         config_changed = true;
 517         break;
 518
 519       case media::H264NALU::kPPS:
 520         last_pps_.assign(nalu.data, nalu.data + nalu.size);
 521         config_changed = true;
 522         result = parser_.ParsePPS(&last_pps_id_);
 523         if (result == media::H264Parser::kUnsupportedStream) {
 524           DLOG(ERROR) << "Unsupported PPS";
 525           NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
 526           return;
 527         }
 528         if (result != media::H264Parser::kOk) {
 529           DLOG(ERROR) << "Could not parse PPS";
 530           NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
 531           return;
 532         }
 533         break;
 534
 535       case media::H264NALU::kSliceDataA:
 536       case media::H264NALU::kSliceDataB:
 537       case media::H264NALU::kSliceDataC:
 538       case media::H264NALU::kNonIDRSlice:
 539         // TODO(sandersd): Check that there has been an IDR slice since the
 540         // last reset.
 541       case media::H264NALU::kIDRSlice:
 542         // Compute the |pic_order_cnt| for the picture from the first slice.
 543         // TODO(sandersd): Make sure that any further slices are part of the
 544         // same picture or a redundant coded picture.
 545         if (!has_slice) {
 546           media::H264SliceHeader slice_hdr;
 547           result = parser_.ParseSliceHeader(nalu, &slice_hdr);
 548           if (result == media::H264Parser::kUnsupportedStream) {
 549             DLOG(ERROR) << "Unsupported slice header";
 550             NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
 551             return;
 552           }
 553           if (result != media::H264Parser::kOk) {
 554             DLOG(ERROR) << "Could not parse slice header";
 555             NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
 556             return;
 557           }
 558
 559           // TODO(sandersd): Maintain a cache of configurations and reconfigure
 560           // only when a slice references a new config.
 561           DCHECK_EQ(slice_hdr.pic_parameter_set_id, last_pps_id_);
 562           const media::H264PPS* pps =
 563               parser_.GetPPS(slice_hdr.pic_parameter_set_id);
 564           if (!pps) {
 565             DLOG(ERROR) << "Mising PPS referenced by slice";
 566             NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
 567             return;
 568           }
 569
 570           DCHECK_EQ(pps->seq_parameter_set_id, last_sps_id_);
 571           const media::H264SPS* sps = parser_.GetSPS(pps->seq_parameter_set_id);
 572           if (!sps) {
 573             DLOG(ERROR) << "Mising SPS referenced by PPS";
 574             NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
 575             return;
 576           }
 577
 578           if (!poc_.ComputePicOrderCnt(sps, slice_hdr, &frame->pic_order_cnt)) {
 579             DLOG(ERROR) << "Unable to compute POC";
 580             NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
 581             return;
 582           }
 583
 584           if (sps->vui_parameters_present_flag &&
 585               sps->bitstream_restriction_flag) {
 586             frame->reorder_window = std::min(sps->max_num_reorder_frames,
 587                                              kMaxReorderQueueSize - 1);
 588           }
 589         }
 590         has_slice = true;
 591       default:
 592         nalus.push_back(nalu);
 593         data_size += kNALUHeaderLength + nalu.size;
 594         break;
 595     }
 596   }
 597
 598   // Initialize VideoToolbox.
 599   // TODO(sandersd): Instead of assuming that the last SPS and PPS units are
 600   // always the correct ones, maintain a cache of recent SPS and PPS units and
 601   // select from them using the slice header.
 602   if (config_changed) {
 603     if (last_sps_.size() == 0 || last_pps_.size() == 0) {
 604       DLOG(ERROR) << "Invalid configuration data";
 605       NotifyError(INVALID_ARGUMENT, SFT_INVALID_STREAM);
 606       return;
 607     }
 608     if (!ConfigureDecoder())
 609       return;
 610   }
 611
 612   // If there are no image slices, drop the bitstream buffer by returning an
 613   // empty frame.
 614   if (!has_slice) {
 615     if (!FinishDelayedFrames())
 616       return;
 617     gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
 618         &VTVideoDecodeAccelerator::DecodeDone, weak_this_, frame));
 619     return;
 620   }
 621
 622   // If the session is not configured by this point, fail.
 623   if (!session_) {
 624     DLOG(ERROR) << "Configuration data missing";
 625     NotifyError(INVALID_ARGUMENT, SFT_INVALID_STREAM);
 626     return;
 627   }
 628
 629   // Update the frame metadata with configuration data.
 630   frame->coded_size = coded_size_;
 631
 632   // Create a memory-backed CMBlockBuffer for the translated data.
 633   // TODO(sandersd): Pool of memory blocks.
 634   base::ScopedCFTypeRef<CMBlockBufferRef> data;
 635   OSStatus status = CMBlockBufferCreateWithMemoryBlock(
 636       kCFAllocatorDefault,
 637       nullptr,              // &memory_block
 638       data_size,            // block_length
 639       kCFAllocatorDefault,  // block_allocator
 640       nullptr,              // &custom_block_source
 641       0,                    // offset_to_data
 642       data_size,            // data_length
 643       0,                    // flags
 644       data.InitializeInto());
 645   if (status) {
 646     NOTIFY_STATUS("CMBlockBufferCreateWithMemoryBlock()", status,
 647                   SFT_PLATFORM_ERROR);
 648     return;
 649   }
 650
 651   // Copy NALU data into the CMBlockBuffer, inserting length headers.
 652   size_t offset = 0;
 653   for (size_t i = 0; i < nalus.size(); i++) {
 654     media::H264NALU& nalu = nalus[i];
 655     uint32_t header = base::HostToNet32(static_cast<uint32_t>(nalu.size));
 656     status = CMBlockBufferReplaceDataBytes(
 657         &header, data, offset, kNALUHeaderLength);
 658     if (status) {
 659       NOTIFY_STATUS("CMBlockBufferReplaceDataBytes()", status,
 660                     SFT_PLATFORM_ERROR);
 661       return;
 662     }
 663     offset += kNALUHeaderLength;
 664     status = CMBlockBufferReplaceDataBytes(nalu.data, data, offset, nalu.size);
 665     if (status) {
 666       NOTIFY_STATUS("CMBlockBufferReplaceDataBytes()", status,
 667                     SFT_PLATFORM_ERROR);
 668       return;
 669     }
 670     offset += nalu.size;
 671   }
 672
 673   // Package the data in a CMSampleBuffer.
 674   base::ScopedCFTypeRef<CMSampleBufferRef> sample;
 675   status = CMSampleBufferCreate(
 676       kCFAllocatorDefault,
 677       data,                 // data_buffer
 678       true,                 // data_ready
 679       nullptr,              // make_data_ready_callback
 680       nullptr,              // make_data_ready_refcon
 681       format_,              // format_description
 682       1,                    // num_samples
 683       0,                    // num_sample_timing_entries
 684       nullptr,              // &sample_timing_array
 685       0,                    // num_sample_size_entries
 686       nullptr,              // &sample_size_array
 687       sample.InitializeInto());
 688   if (status) {
 689     NOTIFY_STATUS("CMSampleBufferCreate()", status, SFT_PLATFORM_ERROR);
 690     return;
 691   }
 692
 693   // Send the frame for decoding.
 694   // Asynchronous Decompression allows for parallel submission of frames
 695   // (without it, DecodeFrame() does not return until the frame has been
 696   // decoded). We don't enable Temporal Processing so that frames are always
 697   // returned in decode order; this makes it easier to avoid deadlock.
 698   VTDecodeFrameFlags decode_flags =
 699       kVTDecodeFrame_EnableAsynchronousDecompression;
 700   status = VTDecompressionSessionDecodeFrame(
 701       session_,
 702       sample,                                 // sample_buffer
 703       decode_flags,                           // decode_flags
 704       reinterpret_cast<void*>(frame),         // source_frame_refcon
 705       nullptr);                               // &info_flags_out
 706   if (status) {
 707     NOTIFY_STATUS("VTDecompressionSessionDecodeFrame()", status,
 708                   SFT_DECODE_ERROR);
 709     return;
 710   }
 711 }
 712
 713 // This method may be called on any VideoToolbox thread.
 714 void VTVideoDecodeAccelerator::Output(
 715     void* source_frame_refcon,
 716     OSStatus status,
 717     CVImageBufferRef image_buffer) {
 718   if (status) {
 719     NOTIFY_STATUS("Decoding", status, SFT_DECODE_ERROR);
 720     return;
 721   }
 722
 723   // The type of |image_buffer| is CVImageBuffer, but we only handle
 724   // CVPixelBuffers. This should be guaranteed as we set
 725   // kCVPixelBufferOpenGLCompatibilityKey in |image_config|.
 726   //
 727   // Sometimes, for unknown reasons (http://crbug.com/453050), |image_buffer| is
 728   // NULL, which causes CFGetTypeID() to crash. While the rest of the code would
 729   // smoothly handle NULL as a dropped frame, we choose to fail permanantly here
 730   // until the issue is better understood.
 731   if (!image_buffer || CFGetTypeID(image_buffer) != CVPixelBufferGetTypeID()) {
 732     DLOG(ERROR) << "Decoded frame is not a CVPixelBuffer";
 733     NotifyError(PLATFORM_FAILURE, SFT_DECODE_ERROR);
 734     return;
 735   }
 736
 737   Frame* frame = reinterpret_cast<Frame*>(source_frame_refcon);
 738   frame->image.reset(image_buffer, base::scoped_policy::RETAIN);
 739   gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
 740       &VTVideoDecodeAccelerator::DecodeDone, weak_this_, frame));
 741 }
 742
 743 void VTVideoDecodeAccelerator::DecodeDone(Frame* frame) {
 744   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 745   DCHECK_EQ(1u, pending_frames_.count(frame->bitstream_id));
 746   Task task(TASK_FRAME);
 747   task.frame = pending_frames_[frame->bitstream_id];
 748   pending_frames_.erase(frame->bitstream_id);
 749   task_queue_.push(task);
 750   ProcessWorkQueues();
 751 }
 752
 753 void VTVideoDecodeAccelerator::FlushTask(TaskType type) {
 754   DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
 755   FinishDelayedFrames();
 756
 757   // Always queue a task, even if FinishDelayedFrames() fails, so that
 758   // destruction always completes.
 759   gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
 760       &VTVideoDecodeAccelerator::FlushDone, weak_this_, type));
 761 }
 762
 763 void VTVideoDecodeAccelerator::FlushDone(TaskType type) {
 764   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 765   task_queue_.push(Task(type));
 766   ProcessWorkQueues();
 767 }
 768
 769 void VTVideoDecodeAccelerator::Decode(const media::BitstreamBuffer& bitstream) {
 770   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 771   DCHECK_EQ(0u, assigned_bitstream_ids_.count(bitstream.id()));
 772   assigned_bitstream_ids_.insert(bitstream.id());
 773   Frame* frame = new Frame(bitstream.id());
 774   pending_frames_[frame->bitstream_id] = make_linked_ptr(frame);
 775   decoder_thread_.message_loop_proxy()->PostTask(FROM_HERE, base::Bind(
 776       &VTVideoDecodeAccelerator::DecodeTask, base::Unretained(this),
 777       bitstream, frame));
 778 }
 779
 780 void VTVideoDecodeAccelerator::AssignPictureBuffers(
 781     const std::vector<media::PictureBuffer>& pictures) {
 782   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 783
 784   for (const media::PictureBuffer& picture : pictures) {
 785     DCHECK(!texture_ids_.count(picture.id()));
 786     assigned_picture_ids_.insert(picture.id());
 787     available_picture_ids_.push_back(picture.id());
 788     texture_ids_[picture.id()] = picture.texture_id();
 789   }
 790
 791   // Pictures are not marked as uncleared until after this method returns, and
 792   // they will be broken if they are used before that happens. So, schedule
 793   // future work after that happens.
 794   gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
 795       &VTVideoDecodeAccelerator::ProcessWorkQueues, weak_this_));
 796 }
 797
 798 void VTVideoDecodeAccelerator::ReusePictureBuffer(int32_t picture_id) {
 799   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 800   DCHECK_EQ(CFGetRetainCount(picture_bindings_[picture_id]), 1);
 801   picture_bindings_.erase(picture_id);
 802   if (assigned_picture_ids_.count(picture_id) != 0) {
 803     available_picture_ids_.push_back(picture_id);
 804     ProcessWorkQueues();
 805   } else {
 806     client_->DismissPictureBuffer(picture_id);
 807   }
 808 }
 809
 810 void VTVideoDecodeAccelerator::ProcessWorkQueues() {
 811   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 812   switch (state_) {
 813     case STATE_DECODING:
 814       // TODO(sandersd): Batch where possible.
 815       while (state_ == STATE_DECODING) {
 816         if (!ProcessReorderQueue() && !ProcessTaskQueue())
 817           break;
 818       }
 819       return;
 820
 821     case STATE_ERROR:
 822       // Do nothing until Destroy() is called.
 823       return;
 824
 825     case STATE_DESTROYING:
 826       // Drop tasks until we are ready to destruct.
 827       while (!task_queue_.empty()) {
 828         if (task_queue_.front().type == TASK_DESTROY) {
 829           delete this;
 830           return;
 831         }
 832         task_queue_.pop();
 833       }
 834       return;
 835   }
 836 }
 837
 838 bool VTVideoDecodeAccelerator::ProcessTaskQueue() {
 839   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 840   DCHECK_EQ(state_, STATE_DECODING);
 841
 842   if (task_queue_.empty())
 843     return false;
 844
 845   const Task& task = task_queue_.front();
 846   switch (task.type) {
 847     case TASK_FRAME:
 848       // TODO(sandersd): Signal IDR explicitly (not using pic_order_cnt == 0).
 849       if (reorder_queue_.size() < kMaxReorderQueueSize &&
 850           (task.frame->pic_order_cnt != 0 || reorder_queue_.empty())) {
 851         assigned_bitstream_ids_.erase(task.frame->bitstream_id);
 852         client_->NotifyEndOfBitstreamBuffer(task.frame->bitstream_id);
 853         reorder_queue_.push(task.frame);
 854         task_queue_.pop();
 855         return true;
 856       }
 857       return false;
 858
 859     case TASK_FLUSH:
 860       DCHECK_EQ(task.type, pending_flush_tasks_.front());
 861       if (reorder_queue_.size() == 0) {
 862         pending_flush_tasks_.pop();
 863         client_->NotifyFlushDone();
 864         task_queue_.pop();
 865         return true;
 866       }
 867       return false;
 868
 869     case TASK_RESET:
 870       DCHECK_EQ(task.type, pending_flush_tasks_.front());
 871       if (reorder_queue_.size() == 0) {
 872         last_sps_id_ = -1;
 873         last_pps_id_ = -1;
 874         last_sps_.clear();
 875         last_spsext_.clear();
 876         last_pps_.clear();
 877         poc_.Reset();
 878         pending_flush_tasks_.pop();
 879         client_->NotifyResetDone();
 880         task_queue_.pop();
 881         return true;
 882       }
 883       return false;
 884
 885     case TASK_DESTROY:
 886       NOTREACHED() << "Can't destroy while in STATE_DECODING.";
 887       NotifyError(ILLEGAL_STATE, SFT_PLATFORM_ERROR);
 888       return false;
 889   }
 890 }
 891
 892 bool VTVideoDecodeAccelerator::ProcessReorderQueue() {
 893   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 894   DCHECK_EQ(state_, STATE_DECODING);
 895
 896   if (reorder_queue_.empty())
 897     return false;
 898
 899   // If the next task is a flush (because there is a pending flush or becuase
 900   // the next frame is an IDR), then we don't need a full reorder buffer to send
 901   // the next frame.
 902   bool flushing = !task_queue_.empty() &&
 903                   (task_queue_.front().type != TASK_FRAME ||
 904                    task_queue_.front().frame->pic_order_cnt == 0);
 905
 906   size_t reorder_window = std::max(0, reorder_queue_.top()->reorder_window);
 907   if (flushing || reorder_queue_.size() > reorder_window) {
 908     if (ProcessFrame(*reorder_queue_.top())) {
 909       reorder_queue_.pop();
 910       return true;
 911     }
 912   }
 913
 914   return false;
 915 }
 916
 917 bool VTVideoDecodeAccelerator::ProcessFrame(const Frame& frame) {
 918   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 919   DCHECK_EQ(state_, STATE_DECODING);
 920
 921   // If the next pending flush is for a reset, then the frame will be dropped.
 922   bool resetting = !pending_flush_tasks_.empty() &&
 923                    pending_flush_tasks_.front() == TASK_RESET;
 924
 925   if (!resetting && frame.image.get()) {
 926     // If the |coded_size| has changed, request new picture buffers and then
 927     // wait for them.
 928     // TODO(sandersd): If GpuVideoDecoder didn't specifically check the size of
 929     // textures, this would be unnecessary, as the size is actually a property
 930     // of the texture binding, not the texture. We rebind every frame, so the
 931     // size passed to ProvidePictureBuffers() is meaningless.
 932     if (picture_size_ != frame.coded_size) {
 933       // Dismiss current pictures.
 934       for (int32_t picture_id : assigned_picture_ids_)
 935         client_->DismissPictureBuffer(picture_id);
 936       assigned_picture_ids_.clear();
 937       available_picture_ids_.clear();
 938
 939       // Request new pictures.
 940       picture_size_ = frame.coded_size;
 941       client_->ProvidePictureBuffers(
 942           kNumPictureBuffers, coded_size_, GL_TEXTURE_RECTANGLE_ARB);
 943       return false;
 944     }
 945     if (!SendFrame(frame))
 946       return false;
 947   }
 948
 949   return true;
 950 }
 951
 952 bool VTVideoDecodeAccelerator::SendFrame(const Frame& frame) {
 953   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 954   DCHECK_EQ(state_, STATE_DECODING);
 955
 956   if (available_picture_ids_.empty())
 957     return false;
 958
 959   int32_t picture_id = available_picture_ids_.back();
 960   IOSurfaceRef surface = CVPixelBufferGetIOSurface(frame.image.get());
 961
 962   if (!make_context_current_.Run()) {
 963     DLOG(ERROR) << "Failed to make GL context current";
 964     NotifyError(PLATFORM_FAILURE, SFT_PLATFORM_ERROR);
 965     return false;
 966   }
 967
 968   glEnable(GL_TEXTURE_RECTANGLE_ARB);
 969   gfx::ScopedTextureBinder
 970       texture_binder(GL_TEXTURE_RECTANGLE_ARB, texture_ids_[picture_id]);
 971   CGLError status = CGLTexImageIOSurface2D(
 972       cgl_context_,                 // ctx
 973       GL_TEXTURE_RECTANGLE_ARB,     // target
 974       GL_RGB,                       // internal_format
 975       frame.coded_size.width(),     // width
 976       frame.coded_size.height(),    // height
 977       GL_YCBCR_422_APPLE,           // format
 978       GL_UNSIGNED_SHORT_8_8_APPLE,  // type
 979       surface,                      // io_surface
 980       0);                           // plane
 981   if (status != kCGLNoError) {
 982     NOTIFY_STATUS("CGLTexImageIOSurface2D()", status, SFT_PLATFORM_ERROR);
 983     return false;
 984   }
 985   glDisable(GL_TEXTURE_RECTANGLE_ARB);
 986
 987   available_picture_ids_.pop_back();
 988   picture_bindings_[picture_id] = frame.image;
 989   client_->PictureReady(media::Picture(picture_id, frame.bitstream_id,
 990                                        gfx::Rect(frame.coded_size), false));
 991   return true;
 992 }
 993
 994 void VTVideoDecodeAccelerator::NotifyError(
 995     Error vda_error_type,
 996     VTVDASessionFailureType session_failure_type) {
 997   DCHECK_LT(session_failure_type, SFT_MAX + 1);
 998   if (!gpu_thread_checker_.CalledOnValidThread()) {
 999     gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
1000         &VTVideoDecodeAccelerator::NotifyError, weak_this_, vda_error_type,
1001         session_failure_type));
1002   } else if (state_ == STATE_DECODING) {
1003     state_ = STATE_ERROR;
1004     UMA_HISTOGRAM_ENUMERATION("Media.VTVDA.SessionFailureReason",
1005                               session_failure_type,
1006                               SFT_MAX + 1);
1007     client_->NotifyError(vda_error_type);
1008   }
1009 }
1010
1011 void VTVideoDecodeAccelerator::QueueFlush(TaskType type) {
1012   DCHECK(gpu_thread_checker_.CalledOnValidThread());
1013   pending_flush_tasks_.push(type);
1014   decoder_thread_.message_loop_proxy()->PostTask(FROM_HERE, base::Bind(
1015       &VTVideoDecodeAccelerator::FlushTask, base::Unretained(this),
1016       type));
1017
1018   // If this is a new flush request, see if we can make progress.
1019   if (pending_flush_tasks_.size() == 1)
1020     ProcessWorkQueues();
1021 }
1022
1023 void VTVideoDecodeAccelerator::Flush() {
1024   DCHECK(gpu_thread_checker_.CalledOnValidThread());
1025   QueueFlush(TASK_FLUSH);
1026 }
1027
1028 void VTVideoDecodeAccelerator::Reset() {
1029   DCHECK(gpu_thread_checker_.CalledOnValidThread());
1030   QueueFlush(TASK_RESET);
1031 }
1032
1033 void VTVideoDecodeAccelerator::Destroy() {
1034   DCHECK(gpu_thread_checker_.CalledOnValidThread());
1035
1036   // In a forceful shutdown, the decoder thread may be dead already.
1037   if (!decoder_thread_.IsRunning()) {
1038     delete this;
1039     return;
1040   }
1041
1042   // For a graceful shutdown, return assigned buffers and flush before
1043   // destructing |this|.
1044   // TODO(sandersd): Make sure the decoder won't try to read the buffers again
1045   // before discarding them.
1046   for (int32_t bitstream_id : assigned_bitstream_ids_)
1047     client_->NotifyEndOfBitstreamBuffer(bitstream_id);
1048   assigned_bitstream_ids_.clear();
1049   state_ = STATE_DESTROYING;
1050   QueueFlush(TASK_DESTROY);
1051 }
1052
1053 bool VTVideoDecodeAccelerator::CanDecodeOnIOThread() {
1054   return false;
1055 }
1056
1057 }  // namespace content