content/common/gpu/media/vt_video_decode_accelerator.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <algorithm>
   6
   7 #include <CoreVideo/CoreVideo.h>
   8 #include <OpenGL/CGLIOSurface.h>
   9 #include <OpenGL/gl.h>
  10
  11 #include "base/bind.h"
  12 #include "base/command_line.h"
  13 #include "base/logging.h"
  14 #include "base/mac/mac_logging.h"
  15 #include "base/metrics/histogram_macros.h"
  16 #include "base/sys_byteorder.h"
  17 #include "base/thread_task_runner_handle.h"
  18 #include "content/common/gpu/media/vt_video_decode_accelerator.h"
  19 #include "content/public/common/content_switches.h"
  20 #include "media/base/limits.h"
  21 #include "ui/gl/scoped_binders.h"
  22
  23 using content_common_gpu_media::kModuleVt;
  24 using content_common_gpu_media::InitializeStubs;
  25 using content_common_gpu_media::IsVtInitialized;
  26 using content_common_gpu_media::StubPathMap;
  27
  28 #define NOTIFY_STATUS(name, status, session_failure)   \
  29     do {                                               \
  30       OSSTATUS_DLOG(ERROR, status) << name;            \
  31       NotifyError(PLATFORM_FAILURE, session_failure);  \
  32     } while (0)
  33
  34 namespace content {
  35
  36 // Only H.264 with 4:2:0 chroma sampling is supported.
  37 static const media::VideoCodecProfile kSupportedProfiles[] = {
  38   media::H264PROFILE_BASELINE,
  39   media::H264PROFILE_MAIN,
  40   media::H264PROFILE_EXTENDED,
  41   media::H264PROFILE_HIGH,
  42   media::H264PROFILE_HIGH10PROFILE,
  43   media::H264PROFILE_SCALABLEBASELINE,
  44   media::H264PROFILE_SCALABLEHIGH,
  45   media::H264PROFILE_STEREOHIGH,
  46   media::H264PROFILE_MULTIVIEWHIGH,
  47 };
  48
  49 // Size to use for NALU length headers in AVC format (can be 1, 2, or 4).
  50 static const int kNALUHeaderLength = 4;
  51
  52 // We request 5 picture buffers from the client, each of which has a texture ID
  53 // that we can bind decoded frames to. We need enough to satisfy preroll, and
  54 // enough to avoid unnecessary stalling, but no more than that. The resource
  55 // requirements are low, as we don't need the textures to be backed by storage.
  56 static const int kNumPictureBuffers = media::limits::kMaxVideoFrames + 1;
  57
  58 // Maximum number of frames to queue for reordering before we stop asking for
  59 // more. (NotifyEndOfBitstreamBuffer() is called when frames are moved into the
  60 // reorder queue.)
  61 static const int kMaxReorderQueueSize = 16;
  62
  63 // Build an |image_config| dictionary for VideoToolbox initialization.
  64 static base::ScopedCFTypeRef<CFMutableDictionaryRef>
  65 BuildImageConfig(CMVideoDimensions coded_dimensions) {
  66   base::ScopedCFTypeRef<CFMutableDictionaryRef> image_config;
  67
  68   // TODO(sandersd): Does it save some work or memory to use 4:2:0?
  69   int32_t pixel_format = kCVPixelFormatType_422YpCbCr8;
  70 #define CFINT(i) CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &i)
  71   base::ScopedCFTypeRef<CFNumberRef> cf_pixel_format(CFINT(pixel_format));
  72   base::ScopedCFTypeRef<CFNumberRef> cf_width(CFINT(coded_dimensions.width));
  73   base::ScopedCFTypeRef<CFNumberRef> cf_height(CFINT(coded_dimensions.height));
  74 #undef CFINT
  75   if (!cf_pixel_format.get() || !cf_width.get() || !cf_height.get())
  76     return image_config;
  77
  78   image_config.reset(
  79       CFDictionaryCreateMutable(
  80           kCFAllocatorDefault,
  81           4,  // capacity
  82           &kCFTypeDictionaryKeyCallBacks,
  83           &kCFTypeDictionaryValueCallBacks));
  84   if (!image_config.get())
  85     return image_config;
  86
  87   CFDictionarySetValue(image_config, kCVPixelBufferPixelFormatTypeKey,
  88                        cf_pixel_format);
  89   CFDictionarySetValue(image_config, kCVPixelBufferWidthKey, cf_width);
  90   CFDictionarySetValue(image_config, kCVPixelBufferHeightKey, cf_height);
  91   CFDictionarySetValue(image_config, kCVPixelBufferOpenGLCompatibilityKey,
  92                        kCFBooleanTrue);
  93
  94   return image_config;
  95 }
  96
  97 // Create a VTDecompressionSession using the provided |pps| and |sps|. If
  98 // |require_hardware| is true, the session must uses real hardware decoding
  99 // (as opposed to software decoding inside of VideoToolbox) to be considered
 100 // successful.
 101 //
 102 // TODO(sandersd): Merge with ConfigureDecoder(), as the code is very similar.
 103 static bool CreateVideoToolboxSession(const uint8_t* sps, size_t sps_size,
 104                                       const uint8_t* pps, size_t pps_size,
 105                                       bool require_hardware) {
 106   const uint8_t* data_ptrs[] = {sps, pps};
 107   const size_t data_sizes[] = {sps_size, pps_size};
 108
 109   base::ScopedCFTypeRef<CMFormatDescriptionRef> format;
 110   OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(
 111       kCFAllocatorDefault,
 112       2,                    // parameter_set_count
 113       data_ptrs,            // &parameter_set_pointers
 114       data_sizes,           // &parameter_set_sizes
 115       kNALUHeaderLength,    // nal_unit_header_length
 116       format.InitializeInto());
 117   if (status) {
 118     OSSTATUS_DLOG(WARNING, status)
 119         << "Failed to create CMVideoFormatDescription.";
 120     return false;
 121   }
 122
 123   base::ScopedCFTypeRef<CFMutableDictionaryRef> decoder_config(
 124       CFDictionaryCreateMutable(
 125           kCFAllocatorDefault,
 126           1,  // capacity
 127           &kCFTypeDictionaryKeyCallBacks,
 128           &kCFTypeDictionaryValueCallBacks));
 129   if (!decoder_config.get())
 130     return false;
 131
 132   if (require_hardware) {
 133     CFDictionarySetValue(
 134         decoder_config,
 135         // kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
 136         CFSTR("RequireHardwareAcceleratedVideoDecoder"),
 137         kCFBooleanTrue);
 138   }
 139
 140   base::ScopedCFTypeRef<CFMutableDictionaryRef> image_config(
 141       BuildImageConfig(CMVideoFormatDescriptionGetDimensions(format)));
 142   if (!image_config.get())
 143     return false;
 144
 145   VTDecompressionOutputCallbackRecord callback = {0};
 146
 147   base::ScopedCFTypeRef<VTDecompressionSessionRef> session;
 148   status = VTDecompressionSessionCreate(
 149       kCFAllocatorDefault,
 150       format,               // video_format_description
 151       decoder_config,       // video_decoder_specification
 152       image_config,         // destination_image_buffer_attributes
 153       &callback,            // output_callback
 154       session.InitializeInto());
 155   if (status) {
 156     OSSTATUS_DLOG(WARNING, status) << "Failed to create VTDecompressionSession";
 157     return false;
 158   }
 159
 160   return true;
 161 }
 162
 163 // The purpose of this function is to preload the generic and hardware-specific
 164 // libraries required by VideoToolbox before the GPU sandbox is enabled.
 165 // VideoToolbox normally loads the hardware-specific libraries lazily, so we
 166 // must actually create a decompression session. If creating a decompression
 167 // session fails, hardware decoding will be disabled (Initialize() will always
 168 // return false).
 169 static bool InitializeVideoToolboxInternal() {
 170   if (base::CommandLine::ForCurrentProcess()->HasSwitch(
 171           switches::kDisableAcceleratedVideoDecode)) {
 172     return false;
 173   }
 174
 175   if (!IsVtInitialized()) {
 176     // CoreVideo is also required, but the loader stops after the first path is
 177     // loaded. Instead we rely on the transitive dependency from VideoToolbox to
 178     // CoreVideo.
 179     // TODO(sandersd): Fallback to PrivateFrameworks to support OS X < 10.8.
 180     StubPathMap paths;
 181     paths[kModuleVt].push_back(FILE_PATH_LITERAL(
 182         "/System/Library/Frameworks/VideoToolbox.framework/VideoToolbox"));
 183     if (!InitializeStubs(paths)) {
 184       LOG(WARNING) << "Failed to initialize VideoToolbox framework. "
 185                    << "Hardware accelerated video decoding will be disabled.";
 186       return false;
 187     }
 188   }
 189
 190   // Create a hardware decoding session.
 191   // SPS and PPS data are taken from a 480p sample (buck2.mp4).
 192   const uint8_t sps_normal[] = {0x67, 0x64, 0x00, 0x1e, 0xac, 0xd9, 0x80, 0xd4,
 193                                 0x3d, 0xa1, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00,
 194                                 0x00, 0x03, 0x00, 0x30, 0x8f, 0x16, 0x2d, 0x9a};
 195   const uint8_t pps_normal[] = {0x68, 0xe9, 0x7b, 0xcb};
 196   if (!CreateVideoToolboxSession(sps_normal, arraysize(sps_normal), pps_normal,
 197                                  arraysize(pps_normal), true)) {
 198     LOG(WARNING) << "Failed to create hardware VideoToolbox session. "
 199                  << "Hardware accelerated video decoding will be disabled.";
 200     return false;
 201   }
 202
 203   // Create a software decoding session.
 204   // SPS and PPS data are taken from a 18p sample (small2.mp4).
 205   const uint8_t sps_small[] = {0x67, 0x64, 0x00, 0x0a, 0xac, 0xd9, 0x89, 0x7e,
 206                                0x22, 0x10, 0x00, 0x00, 0x3e, 0x90, 0x00, 0x0e,
 207                                0xa6, 0x08, 0xf1, 0x22, 0x59, 0xa0};
 208   const uint8_t pps_small[] = {0x68, 0xe9, 0x79, 0x72, 0xc0};
 209   if (!CreateVideoToolboxSession(sps_small, arraysize(sps_small), pps_small,
 210                                  arraysize(pps_small), false)) {
 211     LOG(WARNING) << "Failed to create software VideoToolbox session. "
 212                  << "Hardware accelerated video decoding will be disabled.";
 213     return false;
 214   }
 215
 216   return true;
 217 }
 218
 219 bool InitializeVideoToolbox() {
 220   // InitializeVideoToolbox() is called only from the GPU process main thread;
 221   // once for sandbox warmup, and then once each time a VTVideoDecodeAccelerator
 222   // is initialized.
 223   static bool attempted = false;
 224   static bool succeeded = false;
 225
 226   if (!attempted) {
 227     attempted = true;
 228     succeeded = InitializeVideoToolboxInternal();
 229   }
 230
 231   return succeeded;
 232 }
 233
 234 // Route decoded frame callbacks back into the VTVideoDecodeAccelerator.
 235 static void OutputThunk(
 236     void* decompression_output_refcon,
 237     void* source_frame_refcon,
 238     OSStatus status,
 239     VTDecodeInfoFlags info_flags,
 240     CVImageBufferRef image_buffer,
 241     CMTime presentation_time_stamp,
 242     CMTime presentation_duration) {
 243   VTVideoDecodeAccelerator* vda =
 244       reinterpret_cast<VTVideoDecodeAccelerator*>(decompression_output_refcon);
 245   vda->Output(source_frame_refcon, status, image_buffer);
 246 }
 247
 248 VTVideoDecodeAccelerator::Task::Task(TaskType type) : type(type) {
 249 }
 250
 251 VTVideoDecodeAccelerator::Task::~Task() {
 252 }
 253
 254 VTVideoDecodeAccelerator::Frame::Frame(int32_t bitstream_id)
 255     : bitstream_id(bitstream_id), pic_order_cnt(0), reorder_window(0) {
 256 }
 257
 258 VTVideoDecodeAccelerator::Frame::~Frame() {
 259 }
 260
 261 bool VTVideoDecodeAccelerator::FrameOrder::operator()(
 262     const linked_ptr<Frame>& lhs,
 263     const linked_ptr<Frame>& rhs) const {
 264   if (lhs->pic_order_cnt != rhs->pic_order_cnt)
 265     return lhs->pic_order_cnt > rhs->pic_order_cnt;
 266   // If |pic_order_cnt| is the same, fall back on using the bitstream order.
 267   // TODO(sandersd): Assign a sequence number in Decode() and use that instead.
 268   // TODO(sandersd): Using the sequence number, ensure that frames older than
 269   // |kMaxReorderQueueSize| are ordered first, regardless of |pic_order_cnt|.
 270   return lhs->bitstream_id > rhs->bitstream_id;
 271 }
 272
 273 VTVideoDecodeAccelerator::VTVideoDecodeAccelerator(
 274     CGLContextObj cgl_context,
 275     const base::Callback<bool(void)>& make_context_current)
 276     : cgl_context_(cgl_context),
 277       make_context_current_(make_context_current),
 278       client_(nullptr),
 279       state_(STATE_DECODING),
 280       format_(nullptr),
 281       session_(nullptr),
 282       last_sps_id_(-1),
 283       last_pps_id_(-1),
 284       gpu_task_runner_(base::ThreadTaskRunnerHandle::Get()),
 285       decoder_thread_("VTDecoderThread"),
 286       weak_this_factory_(this) {
 287   DCHECK(!make_context_current_.is_null());
 288   callback_.decompressionOutputCallback = OutputThunk;
 289   callback_.decompressionOutputRefCon = this;
 290   weak_this_ = weak_this_factory_.GetWeakPtr();
 291 }
 292
 293 VTVideoDecodeAccelerator::~VTVideoDecodeAccelerator() {
 294 }
 295
 296 bool VTVideoDecodeAccelerator::Initialize(
 297     media::VideoCodecProfile profile,
 298     Client* client) {
 299   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 300   client_ = client;
 301
 302   if (!InitializeVideoToolbox())
 303     return false;
 304
 305   bool profile_supported = false;
 306   for (const auto& supported_profile : kSupportedProfiles) {
 307     if (profile == supported_profile) {
 308       profile_supported = true;
 309       break;
 310     }
 311   }
 312   if (!profile_supported)
 313     return false;
 314
 315   // Spawn a thread to handle parsing and calling VideoToolbox.
 316   if (!decoder_thread_.Start())
 317     return false;
 318
 319   // Count the session as successfully initialized.
 320   UMA_HISTOGRAM_ENUMERATION("Media.VTVDA.SessionFailureReason",
 321                             SFT_SUCCESSFULLY_INITIALIZED,
 322                             SFT_MAX + 1);
 323   return true;
 324 }
 325
 326 bool VTVideoDecodeAccelerator::FinishDelayedFrames() {
 327   DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
 328   if (session_) {
 329     OSStatus status = VTDecompressionSessionWaitForAsynchronousFrames(session_);
 330     if (status) {
 331       NOTIFY_STATUS("VTDecompressionSessionWaitForAsynchronousFrames()",
 332                     status, SFT_PLATFORM_ERROR);
 333       return false;
 334     }
 335   }
 336   return true;
 337 }
 338
 339 bool VTVideoDecodeAccelerator::ConfigureDecoder() {
 340   DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
 341   DCHECK(!last_sps_.empty());
 342   DCHECK(!last_pps_.empty());
 343
 344   // Build the configuration records.
 345   std::vector<const uint8_t*> nalu_data_ptrs;
 346   std::vector<size_t> nalu_data_sizes;
 347   nalu_data_ptrs.reserve(3);
 348   nalu_data_sizes.reserve(3);
 349   nalu_data_ptrs.push_back(&last_sps_.front());
 350   nalu_data_sizes.push_back(last_sps_.size());
 351   if (!last_spsext_.empty()) {
 352     nalu_data_ptrs.push_back(&last_spsext_.front());
 353     nalu_data_sizes.push_back(last_spsext_.size());
 354   }
 355   nalu_data_ptrs.push_back(&last_pps_.front());
 356   nalu_data_sizes.push_back(last_pps_.size());
 357
 358   // Construct a new format description from the parameter sets.
 359   // TODO(sandersd): Replace this with custom code to support OS X < 10.9.
 360   format_.reset();
 361   OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(
 362       kCFAllocatorDefault,
 363       nalu_data_ptrs.size(),      // parameter_set_count
 364       &nalu_data_ptrs.front(),    // &parameter_set_pointers
 365       &nalu_data_sizes.front(),   // &parameter_set_sizes
 366       kNALUHeaderLength,          // nal_unit_header_length
 367       format_.InitializeInto());
 368   if (status) {
 369     NOTIFY_STATUS("CMVideoFormatDescriptionCreateFromH264ParameterSets()",
 370                   status, SFT_PLATFORM_ERROR);
 371     return false;
 372   }
 373
 374   // Store the new configuration data.
 375   CMVideoDimensions coded_dimensions =
 376       CMVideoFormatDescriptionGetDimensions(format_);
 377   coded_size_.SetSize(coded_dimensions.width, coded_dimensions.height);
 378
 379   // If the session is compatible, there's nothing else to do.
 380   if (session_ &&
 381       VTDecompressionSessionCanAcceptFormatDescription(session_, format_)) {
 382     return true;
 383   }
 384
 385   // Prepare VideoToolbox configuration dictionaries.
 386   base::ScopedCFTypeRef<CFMutableDictionaryRef> decoder_config(
 387       CFDictionaryCreateMutable(
 388           kCFAllocatorDefault,
 389           1,  // capacity
 390           &kCFTypeDictionaryKeyCallBacks,
 391           &kCFTypeDictionaryValueCallBacks));
 392   if (!decoder_config.get()) {
 393     DLOG(ERROR) << "Failed to create CFMutableDictionary.";
 394     NotifyError(PLATFORM_FAILURE, SFT_PLATFORM_ERROR);
 395     return false;
 396   }
 397
 398   CFDictionarySetValue(
 399       decoder_config,
 400       // kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder
 401       CFSTR("EnableHardwareAcceleratedVideoDecoder"),
 402       kCFBooleanTrue);
 403
 404   base::ScopedCFTypeRef<CFMutableDictionaryRef> image_config(
 405       BuildImageConfig(coded_dimensions));
 406   if (!image_config.get()) {
 407     DLOG(ERROR) << "Failed to create decoder image configuration.";
 408     NotifyError(PLATFORM_FAILURE, SFT_PLATFORM_ERROR);
 409     return false;
 410   }
 411
 412   // Ensure that the old decoder emits all frames before the new decoder can
 413   // emit any.
 414   if (!FinishDelayedFrames())
 415     return false;
 416
 417   session_.reset();
 418   status = VTDecompressionSessionCreate(
 419       kCFAllocatorDefault,
 420       format_,              // video_format_description
 421       decoder_config,       // video_decoder_specification
 422       image_config,         // destination_image_buffer_attributes
 423       &callback_,           // output_callback
 424       session_.InitializeInto());
 425   if (status) {
 426     NOTIFY_STATUS("VTDecompressionSessionCreate()", status,
 427                   SFT_UNSUPPORTED_STREAM_PARAMETERS);
 428     return false;
 429   }
 430
 431   // Report whether hardware decode is being used.
 432   bool using_hardware = false;
 433   base::ScopedCFTypeRef<CFBooleanRef> cf_using_hardware;
 434   if (VTSessionCopyProperty(
 435           session_,
 436           // kVTDecompressionPropertyKey_UsingHardwareAcceleratedVideoDecoder
 437           CFSTR("UsingHardwareAcceleratedVideoDecoder"),
 438           kCFAllocatorDefault,
 439           cf_using_hardware.InitializeInto()) == 0) {
 440     using_hardware = CFBooleanGetValue(cf_using_hardware);
 441   }
 442   UMA_HISTOGRAM_BOOLEAN("Media.VTVDA.HardwareAccelerated", using_hardware);
 443
 444   return true;
 445 }
 446
 447 void VTVideoDecodeAccelerator::DecodeTask(
 448     const media::BitstreamBuffer& bitstream,
 449     Frame* frame) {
 450   DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
 451
 452   // Map the bitstream buffer.
 453   base::SharedMemory memory(bitstream.handle(), true);
 454   size_t size = bitstream.size();
 455   if (!memory.Map(size)) {
 456     DLOG(ERROR) << "Failed to map bitstream buffer";
 457     NotifyError(PLATFORM_FAILURE, SFT_PLATFORM_ERROR);
 458     return;
 459   }
 460   const uint8_t* buf = static_cast<uint8_t*>(memory.memory());
 461
 462   // NALUs are stored with Annex B format in the bitstream buffer (start codes),
 463   // but VideoToolbox expects AVC format (length headers), so we must rewrite
 464   // the data.
 465   //
 466   // Locate relevant NALUs and compute the size of the rewritten data. Also
 467   // record any parameter sets for VideoToolbox initialization.
 468   bool config_changed = false;
 469   bool has_slice = false;
 470   size_t data_size = 0;
 471   std::vector<media::H264NALU> nalus;
 472   parser_.SetStream(buf, size);
 473   media::H264NALU nalu;
 474   while (true) {
 475     media::H264Parser::Result result = parser_.AdvanceToNextNALU(&nalu);
 476     if (result == media::H264Parser::kEOStream)
 477       break;
 478     if (result == media::H264Parser::kUnsupportedStream) {
 479       DLOG(ERROR) << "Unsupported H.264 stream";
 480       NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
 481       return;
 482     }
 483     if (result != media::H264Parser::kOk) {
 484       DLOG(ERROR) << "Failed to parse H.264 stream";
 485       NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
 486       return;
 487     }
 488     switch (nalu.nal_unit_type) {
 489       case media::H264NALU::kSPS:
 490         last_sps_.assign(nalu.data, nalu.data + nalu.size);
 491         last_spsext_.clear();
 492         config_changed = true;
 493         result = parser_.ParseSPS(&last_sps_id_);
 494         if (result == media::H264Parser::kUnsupportedStream) {
 495           DLOG(ERROR) << "Unsupported SPS";
 496           NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
 497           return;
 498         }
 499         if (result != media::H264Parser::kOk) {
 500           DLOG(ERROR) << "Could not parse SPS";
 501           NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
 502           return;
 503         }
 504         break;
 505
 506       case media::H264NALU::kSPSExt:
 507         // TODO(sandersd): Check that the previous NALU was an SPS.
 508         last_spsext_.assign(nalu.data, nalu.data + nalu.size);
 509         config_changed = true;
 510         break;
 511
 512       case media::H264NALU::kPPS:
 513         last_pps_.assign(nalu.data, nalu.data + nalu.size);
 514         config_changed = true;
 515         result = parser_.ParsePPS(&last_pps_id_);
 516         if (result == media::H264Parser::kUnsupportedStream) {
 517           DLOG(ERROR) << "Unsupported PPS";
 518           NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
 519           return;
 520         }
 521         if (result != media::H264Parser::kOk) {
 522           DLOG(ERROR) << "Could not parse PPS";
 523           NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
 524           return;
 525         }
 526         break;
 527
 528       case media::H264NALU::kSliceDataA:
 529       case media::H264NALU::kSliceDataB:
 530       case media::H264NALU::kSliceDataC:
 531       case media::H264NALU::kNonIDRSlice:
 532         // TODO(sandersd): Check that there has been an IDR slice since the
 533         // last reset.
 534       case media::H264NALU::kIDRSlice:
 535         // Compute the |pic_order_cnt| for the picture from the first slice.
 536         // TODO(sandersd): Make sure that any further slices are part of the
 537         // same picture or a redundant coded picture.
 538         if (!has_slice) {
 539           media::H264SliceHeader slice_hdr;
 540           result = parser_.ParseSliceHeader(nalu, &slice_hdr);
 541           if (result == media::H264Parser::kUnsupportedStream) {
 542             DLOG(ERROR) << "Unsupported slice header";
 543             NotifyError(PLATFORM_FAILURE, SFT_UNSUPPORTED_STREAM);
 544             return;
 545           }
 546           if (result != media::H264Parser::kOk) {
 547             DLOG(ERROR) << "Could not parse slice header";
 548             NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
 549             return;
 550           }
 551
 552           // TODO(sandersd): Maintain a cache of configurations and reconfigure
 553           // only when a slice references a new config.
 554           DCHECK_EQ(slice_hdr.pic_parameter_set_id, last_pps_id_);
 555           const media::H264PPS* pps =
 556               parser_.GetPPS(slice_hdr.pic_parameter_set_id);
 557           if (!pps) {
 558             DLOG(ERROR) << "Mising PPS referenced by slice";
 559             NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
 560             return;
 561           }
 562
 563           DCHECK_EQ(pps->seq_parameter_set_id, last_sps_id_);
 564           const media::H264SPS* sps = parser_.GetSPS(pps->seq_parameter_set_id);
 565           if (!sps) {
 566             DLOG(ERROR) << "Mising SPS referenced by PPS";
 567             NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
 568             return;
 569           }
 570
 571           if (!poc_.ComputePicOrderCnt(sps, slice_hdr, &frame->pic_order_cnt)) {
 572             DLOG(ERROR) << "Unable to compute POC";
 573             NotifyError(UNREADABLE_INPUT, SFT_INVALID_STREAM);
 574             return;
 575           }
 576
 577           if (sps->vui_parameters_present_flag &&
 578               sps->bitstream_restriction_flag) {
 579             frame->reorder_window = std::min(sps->max_num_reorder_frames,
 580                                              kMaxReorderQueueSize - 1);
 581           }
 582         }
 583         has_slice = true;
 584       default:
 585         nalus.push_back(nalu);
 586         data_size += kNALUHeaderLength + nalu.size;
 587         break;
 588     }
 589   }
 590
 591   // Initialize VideoToolbox.
 592   // TODO(sandersd): Instead of assuming that the last SPS and PPS units are
 593   // always the correct ones, maintain a cache of recent SPS and PPS units and
 594   // select from them using the slice header.
 595   if (config_changed) {
 596     if (last_sps_.size() == 0 || last_pps_.size() == 0) {
 597       DLOG(ERROR) << "Invalid configuration data";
 598       NotifyError(INVALID_ARGUMENT, SFT_INVALID_STREAM);
 599       return;
 600     }
 601     if (!ConfigureDecoder())
 602       return;
 603   }
 604
 605   // If there are no image slices, drop the bitstream buffer by returning an
 606   // empty frame.
 607   if (!has_slice) {
 608     if (!FinishDelayedFrames())
 609       return;
 610     gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
 611         &VTVideoDecodeAccelerator::DecodeDone, weak_this_, frame));
 612     return;
 613   }
 614
 615   // If the session is not configured by this point, fail.
 616   if (!session_) {
 617     DLOG(ERROR) << "Configuration data missing";
 618     NotifyError(INVALID_ARGUMENT, SFT_INVALID_STREAM);
 619     return;
 620   }
 621
 622   // Update the frame metadata with configuration data.
 623   frame->coded_size = coded_size_;
 624
 625   // Create a memory-backed CMBlockBuffer for the translated data.
 626   // TODO(sandersd): Pool of memory blocks.
 627   base::ScopedCFTypeRef<CMBlockBufferRef> data;
 628   OSStatus status = CMBlockBufferCreateWithMemoryBlock(
 629       kCFAllocatorDefault,
 630       nullptr,              // &memory_block
 631       data_size,            // block_length
 632       kCFAllocatorDefault,  // block_allocator
 633       nullptr,              // &custom_block_source
 634       0,                    // offset_to_data
 635       data_size,            // data_length
 636       0,                    // flags
 637       data.InitializeInto());
 638   if (status) {
 639     NOTIFY_STATUS("CMBlockBufferCreateWithMemoryBlock()", status,
 640                   SFT_PLATFORM_ERROR);
 641     return;
 642   }
 643
 644   // Copy NALU data into the CMBlockBuffer, inserting length headers.
 645   size_t offset = 0;
 646   for (size_t i = 0; i < nalus.size(); i++) {
 647     media::H264NALU& nalu = nalus[i];
 648     uint32_t header = base::HostToNet32(static_cast<uint32_t>(nalu.size));
 649     status = CMBlockBufferReplaceDataBytes(
 650         &header, data, offset, kNALUHeaderLength);
 651     if (status) {
 652       NOTIFY_STATUS("CMBlockBufferReplaceDataBytes()", status,
 653                     SFT_PLATFORM_ERROR);
 654       return;
 655     }
 656     offset += kNALUHeaderLength;
 657     status = CMBlockBufferReplaceDataBytes(nalu.data, data, offset, nalu.size);
 658     if (status) {
 659       NOTIFY_STATUS("CMBlockBufferReplaceDataBytes()", status,
 660                     SFT_PLATFORM_ERROR);
 661       return;
 662     }
 663     offset += nalu.size;
 664   }
 665
 666   // Package the data in a CMSampleBuffer.
 667   base::ScopedCFTypeRef<CMSampleBufferRef> sample;
 668   status = CMSampleBufferCreate(
 669       kCFAllocatorDefault,
 670       data,                 // data_buffer
 671       true,                 // data_ready
 672       nullptr,              // make_data_ready_callback
 673       nullptr,              // make_data_ready_refcon
 674       format_,              // format_description
 675       1,                    // num_samples
 676       0,                    // num_sample_timing_entries
 677       nullptr,              // &sample_timing_array
 678       0,                    // num_sample_size_entries
 679       nullptr,              // &sample_size_array
 680       sample.InitializeInto());
 681   if (status) {
 682     NOTIFY_STATUS("CMSampleBufferCreate()", status, SFT_PLATFORM_ERROR);
 683     return;
 684   }
 685
 686   // Send the frame for decoding.
 687   // Asynchronous Decompression allows for parallel submission of frames
 688   // (without it, DecodeFrame() does not return until the frame has been
 689   // decoded). We don't enable Temporal Processing so that frames are always
 690   // returned in decode order; this makes it easier to avoid deadlock.
 691   VTDecodeFrameFlags decode_flags =
 692       kVTDecodeFrame_EnableAsynchronousDecompression;
 693   status = VTDecompressionSessionDecodeFrame(
 694       session_,
 695       sample,                                 // sample_buffer
 696       decode_flags,                           // decode_flags
 697       reinterpret_cast<void*>(frame),         // source_frame_refcon
 698       nullptr);                               // &info_flags_out
 699   if (status) {
 700     NOTIFY_STATUS("VTDecompressionSessionDecodeFrame()", status,
 701                   SFT_DECODE_ERROR);
 702     return;
 703   }
 704 }
 705
 706 // This method may be called on any VideoToolbox thread.
 707 void VTVideoDecodeAccelerator::Output(
 708     void* source_frame_refcon,
 709     OSStatus status,
 710     CVImageBufferRef image_buffer) {
 711   if (status) {
 712     NOTIFY_STATUS("Decoding", status, SFT_DECODE_ERROR);
 713     return;
 714   }
 715
 716   // The type of |image_buffer| is CVImageBuffer, but we only handle
 717   // CVPixelBuffers. This should be guaranteed as we set
 718   // kCVPixelBufferOpenGLCompatibilityKey in |image_config|.
 719   //
 720   // Sometimes, for unknown reasons (http://crbug.com/453050), |image_buffer| is
 721   // NULL, which causes CFGetTypeID() to crash. While the rest of the code would
 722   // smoothly handle NULL as a dropped frame, we choose to fail permanantly here
 723   // until the issue is better understood.
 724   if (!image_buffer || CFGetTypeID(image_buffer) != CVPixelBufferGetTypeID()) {
 725     DLOG(ERROR) << "Decoded frame is not a CVPixelBuffer";
 726     NotifyError(PLATFORM_FAILURE, SFT_DECODE_ERROR);
 727     return;
 728   }
 729
 730   Frame* frame = reinterpret_cast<Frame*>(source_frame_refcon);
 731   frame->image.reset(image_buffer, base::scoped_policy::RETAIN);
 732   gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
 733       &VTVideoDecodeAccelerator::DecodeDone, weak_this_, frame));
 734 }
 735
 736 void VTVideoDecodeAccelerator::DecodeDone(Frame* frame) {
 737   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 738   DCHECK_EQ(1u, pending_frames_.count(frame->bitstream_id));
 739   Task task(TASK_FRAME);
 740   task.frame = pending_frames_[frame->bitstream_id];
 741   pending_frames_.erase(frame->bitstream_id);
 742   task_queue_.push(task);
 743   ProcessWorkQueues();
 744 }
 745
 746 void VTVideoDecodeAccelerator::FlushTask(TaskType type) {
 747   DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
 748   FinishDelayedFrames();
 749
 750   // Always queue a task, even if FinishDelayedFrames() fails, so that
 751   // destruction always completes.
 752   gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
 753       &VTVideoDecodeAccelerator::FlushDone, weak_this_, type));
 754 }
 755
 756 void VTVideoDecodeAccelerator::FlushDone(TaskType type) {
 757   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 758   task_queue_.push(Task(type));
 759   ProcessWorkQueues();
 760 }
 761
 762 void VTVideoDecodeAccelerator::Decode(const media::BitstreamBuffer& bitstream) {
 763   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 764   DCHECK_EQ(0u, assigned_bitstream_ids_.count(bitstream.id()));
 765   assigned_bitstream_ids_.insert(bitstream.id());
 766   Frame* frame = new Frame(bitstream.id());
 767   pending_frames_[frame->bitstream_id] = make_linked_ptr(frame);
 768   decoder_thread_.message_loop_proxy()->PostTask(FROM_HERE, base::Bind(
 769       &VTVideoDecodeAccelerator::DecodeTask, base::Unretained(this),
 770       bitstream, frame));
 771 }
 772
 773 void VTVideoDecodeAccelerator::AssignPictureBuffers(
 774     const std::vector<media::PictureBuffer>& pictures) {
 775   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 776
 777   for (const media::PictureBuffer& picture : pictures) {
 778     DCHECK(!texture_ids_.count(picture.id()));
 779     assigned_picture_ids_.insert(picture.id());
 780     available_picture_ids_.push_back(picture.id());
 781     texture_ids_[picture.id()] = picture.texture_id();
 782   }
 783
 784   // Pictures are not marked as uncleared until after this method returns, and
 785   // they will be broken if they are used before that happens. So, schedule
 786   // future work after that happens.
 787   gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
 788       &VTVideoDecodeAccelerator::ProcessWorkQueues, weak_this_));
 789 }
 790
 791 void VTVideoDecodeAccelerator::ReusePictureBuffer(int32_t picture_id) {
 792   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 793   DCHECK_EQ(CFGetRetainCount(picture_bindings_[picture_id]), 1);
 794   picture_bindings_.erase(picture_id);
 795   if (assigned_picture_ids_.count(picture_id) != 0) {
 796     available_picture_ids_.push_back(picture_id);
 797     ProcessWorkQueues();
 798   } else {
 799     client_->DismissPictureBuffer(picture_id);
 800   }
 801 }
 802
 803 void VTVideoDecodeAccelerator::ProcessWorkQueues() {
 804   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 805   switch (state_) {
 806     case STATE_DECODING:
 807       // TODO(sandersd): Batch where possible.
 808       while (state_ == STATE_DECODING) {
 809         if (!ProcessReorderQueue() && !ProcessTaskQueue())
 810           break;
 811       }
 812       return;
 813
 814     case STATE_ERROR:
 815       // Do nothing until Destroy() is called.
 816       return;
 817
 818     case STATE_DESTROYING:
 819       // Drop tasks until we are ready to destruct.
 820       while (!task_queue_.empty()) {
 821         if (task_queue_.front().type == TASK_DESTROY) {
 822           delete this;
 823           return;
 824         }
 825         task_queue_.pop();
 826       }
 827       return;
 828   }
 829 }
 830
 831 bool VTVideoDecodeAccelerator::ProcessTaskQueue() {
 832   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 833   DCHECK_EQ(state_, STATE_DECODING);
 834
 835   if (task_queue_.empty())
 836     return false;
 837
 838   const Task& task = task_queue_.front();
 839   switch (task.type) {
 840     case TASK_FRAME:
 841       // TODO(sandersd): Signal IDR explicitly (not using pic_order_cnt == 0).
 842       if (reorder_queue_.size() < kMaxReorderQueueSize &&
 843           (task.frame->pic_order_cnt != 0 || reorder_queue_.empty())) {
 844         assigned_bitstream_ids_.erase(task.frame->bitstream_id);
 845         client_->NotifyEndOfBitstreamBuffer(task.frame->bitstream_id);
 846         reorder_queue_.push(task.frame);
 847         task_queue_.pop();
 848         return true;
 849       }
 850       return false;
 851
 852     case TASK_FLUSH:
 853       DCHECK_EQ(task.type, pending_flush_tasks_.front());
 854       if (reorder_queue_.size() == 0) {
 855         pending_flush_tasks_.pop();
 856         client_->NotifyFlushDone();
 857         task_queue_.pop();
 858         return true;
 859       }
 860       return false;
 861
 862     case TASK_RESET:
 863       DCHECK_EQ(task.type, pending_flush_tasks_.front());
 864       if (reorder_queue_.size() == 0) {
 865         last_sps_id_ = -1;
 866         last_pps_id_ = -1;
 867         last_sps_.clear();
 868         last_spsext_.clear();
 869         last_pps_.clear();
 870         poc_.Reset();
 871         pending_flush_tasks_.pop();
 872         client_->NotifyResetDone();
 873         task_queue_.pop();
 874         return true;
 875       }
 876       return false;
 877
 878     case TASK_DESTROY:
 879       NOTREACHED() << "Can't destroy while in STATE_DECODING.";
 880       NotifyError(ILLEGAL_STATE, SFT_PLATFORM_ERROR);
 881       return false;
 882   }
 883 }
 884
 885 bool VTVideoDecodeAccelerator::ProcessReorderQueue() {
 886   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 887   DCHECK_EQ(state_, STATE_DECODING);
 888
 889   if (reorder_queue_.empty())
 890     return false;
 891
 892   // If the next task is a flush (because there is a pending flush or becuase
 893   // the next frame is an IDR), then we don't need a full reorder buffer to send
 894   // the next frame.
 895   bool flushing = !task_queue_.empty() &&
 896                   (task_queue_.front().type != TASK_FRAME ||
 897                    task_queue_.front().frame->pic_order_cnt == 0);
 898
 899   size_t reorder_window = std::max(0, reorder_queue_.top()->reorder_window);
 900   if (flushing || reorder_queue_.size() > reorder_window) {
 901     if (ProcessFrame(*reorder_queue_.top())) {
 902       reorder_queue_.pop();
 903       return true;
 904     }
 905   }
 906
 907   return false;
 908 }
 909
 910 bool VTVideoDecodeAccelerator::ProcessFrame(const Frame& frame) {
 911   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 912   DCHECK_EQ(state_, STATE_DECODING);
 913
 914   // If the next pending flush is for a reset, then the frame will be dropped.
 915   bool resetting = !pending_flush_tasks_.empty() &&
 916                    pending_flush_tasks_.front() == TASK_RESET;
 917
 918   if (!resetting && frame.image.get()) {
 919     // If the |coded_size| has changed, request new picture buffers and then
 920     // wait for them.
 921     // TODO(sandersd): If GpuVideoDecoder didn't specifically check the size of
 922     // textures, this would be unnecessary, as the size is actually a property
 923     // of the texture binding, not the texture. We rebind every frame, so the
 924     // size passed to ProvidePictureBuffers() is meaningless.
 925     if (picture_size_ != frame.coded_size) {
 926       // Dismiss current pictures.
 927       for (int32_t picture_id : assigned_picture_ids_)
 928         client_->DismissPictureBuffer(picture_id);
 929       assigned_picture_ids_.clear();
 930       available_picture_ids_.clear();
 931
 932       // Request new pictures.
 933       picture_size_ = frame.coded_size;
 934       client_->ProvidePictureBuffers(
 935           kNumPictureBuffers, coded_size_, GL_TEXTURE_RECTANGLE_ARB);
 936       return false;
 937     }
 938     if (!SendFrame(frame))
 939       return false;
 940   }
 941
 942   return true;
 943 }
 944
 945 bool VTVideoDecodeAccelerator::SendFrame(const Frame& frame) {
 946   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 947   DCHECK_EQ(state_, STATE_DECODING);
 948
 949   if (available_picture_ids_.empty())
 950     return false;
 951
 952   int32_t picture_id = available_picture_ids_.back();
 953   IOSurfaceRef surface = CVPixelBufferGetIOSurface(frame.image.get());
 954
 955   if (!make_context_current_.Run()) {
 956     DLOG(ERROR) << "Failed to make GL context current";
 957     NotifyError(PLATFORM_FAILURE, SFT_PLATFORM_ERROR);
 958     return false;
 959   }
 960
 961   glEnable(GL_TEXTURE_RECTANGLE_ARB);
 962   gfx::ScopedTextureBinder
 963       texture_binder(GL_TEXTURE_RECTANGLE_ARB, texture_ids_[picture_id]);
 964   CGLError status = CGLTexImageIOSurface2D(
 965       cgl_context_,                 // ctx
 966       GL_TEXTURE_RECTANGLE_ARB,     // target
 967       GL_RGB,                       // internal_format
 968       frame.coded_size.width(),     // width
 969       frame.coded_size.height(),    // height
 970       GL_YCBCR_422_APPLE,           // format
 971       GL_UNSIGNED_SHORT_8_8_APPLE,  // type
 972       surface,                      // io_surface
 973       0);                           // plane
 974   if (status != kCGLNoError) {
 975     NOTIFY_STATUS("CGLTexImageIOSurface2D()", status, SFT_PLATFORM_ERROR);
 976     return false;
 977   }
 978   glDisable(GL_TEXTURE_RECTANGLE_ARB);
 979
 980   available_picture_ids_.pop_back();
 981   picture_bindings_[picture_id] = frame.image;
 982   client_->PictureReady(media::Picture(picture_id, frame.bitstream_id,
 983                                        gfx::Rect(frame.coded_size), false));
 984   return true;
 985 }
 986
 987 void VTVideoDecodeAccelerator::NotifyError(
 988     Error vda_error_type,
 989     VTVDASessionFailureType session_failure_type) {
 990   DCHECK_LT(session_failure_type, SFT_MAX + 1);
 991   if (!gpu_thread_checker_.CalledOnValidThread()) {
 992     gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
 993         &VTVideoDecodeAccelerator::NotifyError, weak_this_, vda_error_type,
 994         session_failure_type));
 995   } else if (state_ == STATE_DECODING) {
 996     state_ = STATE_ERROR;
 997     UMA_HISTOGRAM_ENUMERATION("Media.VTVDA.SessionFailureReason",
 998                               session_failure_type,
 999                               SFT_MAX + 1);
1000     client_->NotifyError(vda_error_type);
1001   }
1002 }
1003
1004 void VTVideoDecodeAccelerator::QueueFlush(TaskType type) {
1005   DCHECK(gpu_thread_checker_.CalledOnValidThread());
1006   pending_flush_tasks_.push(type);
1007   decoder_thread_.message_loop_proxy()->PostTask(FROM_HERE, base::Bind(
1008       &VTVideoDecodeAccelerator::FlushTask, base::Unretained(this),
1009       type));
1010
1011   // If this is a new flush request, see if we can make progress.
1012   if (pending_flush_tasks_.size() == 1)
1013     ProcessWorkQueues();
1014 }
1015
1016 void VTVideoDecodeAccelerator::Flush() {
1017   DCHECK(gpu_thread_checker_.CalledOnValidThread());
1018   QueueFlush(TASK_FLUSH);
1019 }
1020
1021 void VTVideoDecodeAccelerator::Reset() {
1022   DCHECK(gpu_thread_checker_.CalledOnValidThread());
1023   QueueFlush(TASK_RESET);
1024 }
1025
1026 void VTVideoDecodeAccelerator::Destroy() {
1027   DCHECK(gpu_thread_checker_.CalledOnValidThread());
1028
1029   // In a forceful shutdown, the decoder thread may be dead already.
1030   if (!decoder_thread_.IsRunning()) {
1031     delete this;
1032     return;
1033   }
1034
1035   // For a graceful shutdown, return assigned buffers and flush before
1036   // destructing |this|.
1037   // TODO(sandersd): Make sure the decoder won't try to read the buffers again
1038   // before discarding them.
1039   for (int32_t bitstream_id : assigned_bitstream_ids_)
1040     client_->NotifyEndOfBitstreamBuffer(bitstream_id);
1041   assigned_bitstream_ids_.clear();
1042   state_ = STATE_DESTROYING;
1043   QueueFlush(TASK_DESTROY);
1044 }
1045
1046 bool VTVideoDecodeAccelerator::CanDecodeOnIOThread() {
1047   return false;
1048 }
1049
1050 // static
1051 media::VideoDecodeAccelerator::SupportedProfiles
1052 VTVideoDecodeAccelerator::GetSupportedProfiles() {
1053   SupportedProfiles profiles;
1054   for (const auto& supported_profile : kSupportedProfiles) {
1055     SupportedProfile profile;
1056     profile.profile = supported_profile;
1057     profile.min_resolution.SetSize(16, 16);
1058     profile.max_resolution.SetSize(4096, 2160);
1059     profiles.push_back(profile);
1060   }
1061   return profiles;
1062 }
1063
1064 }  // namespace content