media/cast/sender/h264_vt_encoder.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/sender/h264_vt_encoder.h"
   6
   7 #include <string>
   8 #include <vector>
   9
  10 #include "base/big_endian.h"
  11 #include "base/bind.h"
  12 #include "base/bind_helpers.h"
  13 #include "base/location.h"
  14 #include "base/logging.h"
  15 #include "base/macros.h"
  16 #include "base/power_monitor/power_monitor.h"
  17 #include "base/synchronization/lock.h"
  18 #include "media/base/mac/corevideo_glue.h"
  19 #include "media/base/mac/video_frame_mac.h"
  20 #include "media/cast/sender/video_frame_factory.h"
  21
  22 namespace media {
  23 namespace cast {
  24
  25 namespace {
  26
  27 // Container for the associated data of a video frame being processed.
  28 struct InProgressFrameEncode {
  29   const RtpTimestamp rtp_timestamp;
  30   const base::TimeTicks reference_time;
  31   const VideoEncoder::FrameEncodedCallback frame_encoded_callback;
  32
  33   InProgressFrameEncode(RtpTimestamp rtp,
  34                         base::TimeTicks r_time,
  35                         VideoEncoder::FrameEncodedCallback callback)
  36       : rtp_timestamp(rtp),
  37         reference_time(r_time),
  38         frame_encoded_callback(callback) {}
  39 };
  40
  41 base::ScopedCFTypeRef<CFDictionaryRef>
  42 DictionaryWithKeysAndValues(CFTypeRef* keys, CFTypeRef* values, size_t size) {
  43   return base::ScopedCFTypeRef<CFDictionaryRef>(CFDictionaryCreate(
  44       kCFAllocatorDefault, keys, values, size, &kCFTypeDictionaryKeyCallBacks,
  45       &kCFTypeDictionaryValueCallBacks));
  46 }
  47
  48 base::ScopedCFTypeRef<CFDictionaryRef> DictionaryWithKeyValue(CFTypeRef key,
  49                                                               CFTypeRef value) {
  50   CFTypeRef keys[1] = {key};
  51   CFTypeRef values[1] = {value};
  52   return DictionaryWithKeysAndValues(keys, values, 1);
  53 }
  54
  55 base::ScopedCFTypeRef<CFArrayRef> ArrayWithIntegers(const int* v, size_t size) {
  56   std::vector<CFNumberRef> numbers;
  57   numbers.reserve(size);
  58   for (const int* end = v + size; v < end; ++v)
  59     numbers.push_back(CFNumberCreate(nullptr, kCFNumberSInt32Type, v));
  60   base::ScopedCFTypeRef<CFArrayRef> array(CFArrayCreate(
  61       kCFAllocatorDefault, reinterpret_cast<const void**>(&numbers[0]),
  62       numbers.size(), &kCFTypeArrayCallBacks));
  63   for (auto& number : numbers) {
  64     CFRelease(number);
  65   }
  66   return array;
  67 }
  68
  69 template <typename NalSizeType>
  70 void CopyNalsToAnnexB(char* avcc_buffer,
  71                       const size_t avcc_size,
  72                       std::string* annexb_buffer) {
  73   static_assert(sizeof(NalSizeType) == 1 || sizeof(NalSizeType) == 2 ||
  74                     sizeof(NalSizeType) == 4,
  75                 "NAL size type has unsupported size");
  76   static const char startcode_3[3] = {0, 0, 1};
  77   DCHECK(avcc_buffer);
  78   DCHECK(annexb_buffer);
  79   size_t bytes_left = avcc_size;
  80   while (bytes_left > 0) {
  81     DCHECK_GT(bytes_left, sizeof(NalSizeType));
  82     NalSizeType nal_size;
  83     base::ReadBigEndian(avcc_buffer, &nal_size);
  84     bytes_left -= sizeof(NalSizeType);
  85     avcc_buffer += sizeof(NalSizeType);
  86
  87     DCHECK_GE(bytes_left, nal_size);
  88     annexb_buffer->append(startcode_3, sizeof(startcode_3));
  89     annexb_buffer->append(avcc_buffer, nal_size);
  90     bytes_left -= nal_size;
  91     avcc_buffer += nal_size;
  92   }
  93 }
  94
  95 // Copy a H.264 frame stored in a CM sample buffer to an Annex B buffer. Copies
  96 // parameter sets for keyframes before the frame data as well.
  97 void CopySampleBufferToAnnexBBuffer(CoreMediaGlue::CMSampleBufferRef sbuf,
  98                                     std::string* annexb_buffer,
  99                                     bool keyframe) {
 100   // Perform two pass, one to figure out the total output size, and another to
 101   // copy the data after having performed a single output allocation. Note that
 102   // we'll allocate a bit more because we'll count 4 bytes instead of 3 for
 103   // video NALs.
 104
 105   OSStatus status;
 106
 107   // Get the sample buffer's block buffer and format description.
 108   auto bb = CoreMediaGlue::CMSampleBufferGetDataBuffer(sbuf);
 109   DCHECK(bb);
 110   auto fdesc = CoreMediaGlue::CMSampleBufferGetFormatDescription(sbuf);
 111   DCHECK(fdesc);
 112
 113   size_t bb_size = CoreMediaGlue::CMBlockBufferGetDataLength(bb);
 114   size_t total_bytes = bb_size;
 115
 116   size_t pset_count;
 117   int nal_size_field_bytes;
 118   status = CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
 119       fdesc, 0, nullptr, nullptr, &pset_count, &nal_size_field_bytes);
 120   if (status ==
 121       CoreMediaGlue::kCMFormatDescriptionBridgeError_InvalidParameter) {
 122     DLOG(WARNING) << " assuming 2 parameter sets and 4 bytes NAL length header";
 123     pset_count = 2;
 124     nal_size_field_bytes = 4;
 125   } else if (status != noErr) {
 126     DLOG(ERROR)
 127         << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: "
 128         << status;
 129     return;
 130   }
 131
 132   if (keyframe) {
 133     const uint8_t* pset;
 134     size_t pset_size;
 135     for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) {
 136       status =
 137           CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
 138               fdesc, pset_i, &pset, &pset_size, nullptr, nullptr);
 139       if (status != noErr) {
 140         DLOG(ERROR)
 141             << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: "
 142             << status;
 143         return;
 144       }
 145       total_bytes += pset_size + nal_size_field_bytes;
 146     }
 147   }
 148
 149   annexb_buffer->reserve(total_bytes);
 150
 151   // Copy all parameter sets before keyframes.
 152   if (keyframe) {
 153     const uint8_t* pset;
 154     size_t pset_size;
 155     for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) {
 156       status =
 157           CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
 158               fdesc, pset_i, &pset, &pset_size, nullptr, nullptr);
 159       if (status != noErr) {
 160         DLOG(ERROR)
 161             << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: "
 162             << status;
 163         return;
 164       }
 165       static const char startcode_4[4] = {0, 0, 0, 1};
 166       annexb_buffer->append(startcode_4, sizeof(startcode_4));
 167       annexb_buffer->append(reinterpret_cast<const char*>(pset), pset_size);
 168     }
 169   }
 170
 171   // Block buffers can be composed of non-contiguous chunks. For the sake of
 172   // keeping this code simple, flatten non-contiguous block buffers.
 173   base::ScopedCFTypeRef<CoreMediaGlue::CMBlockBufferRef> contiguous_bb(
 174       bb, base::scoped_policy::RETAIN);
 175   if (!CoreMediaGlue::CMBlockBufferIsRangeContiguous(bb, 0, 0)) {
 176     contiguous_bb.reset();
 177     status = CoreMediaGlue::CMBlockBufferCreateContiguous(
 178         kCFAllocatorDefault, bb, kCFAllocatorDefault, nullptr, 0, 0, 0,
 179         contiguous_bb.InitializeInto());
 180     if (status != noErr) {
 181       DLOG(ERROR) << " CMBlockBufferCreateContiguous failed: " << status;
 182       return;
 183     }
 184   }
 185
 186   // Copy all the NAL units. In the process convert them from AVCC format
 187   // (length header) to AnnexB format (start code).
 188   char* bb_data;
 189   status = CoreMediaGlue::CMBlockBufferGetDataPointer(contiguous_bb, 0, nullptr,
 190                                                       nullptr, &bb_data);
 191   if (status != noErr) {
 192     DLOG(ERROR) << " CMBlockBufferGetDataPointer failed: " << status;
 193     return;
 194   }
 195
 196   if (nal_size_field_bytes == 1) {
 197     CopyNalsToAnnexB<uint8_t>(bb_data, bb_size, annexb_buffer);
 198   } else if (nal_size_field_bytes == 2) {
 199     CopyNalsToAnnexB<uint16_t>(bb_data, bb_size, annexb_buffer);
 200   } else if (nal_size_field_bytes == 4) {
 201     CopyNalsToAnnexB<uint32_t>(bb_data, bb_size, annexb_buffer);
 202   } else {
 203     NOTREACHED();
 204   }
 205 }
 206
 207 }  // namespace
 208
 209 class H264VideoToolboxEncoder::VideoFrameFactoryImpl
 210     : public base::RefCountedThreadSafe<VideoFrameFactoryImpl>,
 211       public VideoFrameFactory {
 212  public:
 213   // Type that proxies the VideoFrameFactory interface to this class.
 214   class Proxy;
 215
 216   VideoFrameFactoryImpl(const base::WeakPtr<H264VideoToolboxEncoder>& encoder,
 217                         const scoped_refptr<CastEnvironment>& cast_environment)
 218       : encoder_(encoder), cast_environment_(cast_environment) {}
 219
 220   scoped_refptr<VideoFrame> MaybeCreateFrame(
 221       const gfx::Size& frame_size,
 222       base::TimeDelta timestamp) final {
 223     if (frame_size.IsEmpty()) {
 224       DVLOG(1) << "Rejecting empty video frame.";
 225       return nullptr;
 226     }
 227
 228     base::AutoLock auto_lock(lock_);
 229
 230     // If the pool size does not match, speculatively reset the encoder to use
 231     // the new size and return null. Cache the new frame size right away and
 232     // toss away the pixel buffer pool to avoid spurious tasks until the encoder
 233     // is done resetting.
 234     if (frame_size != pool_frame_size_) {
 235       DVLOG(1) << "MaybeCreateFrame: Detected frame size change.";
 236       cast_environment_->PostTask(
 237           CastEnvironment::MAIN, FROM_HERE,
 238           base::Bind(&H264VideoToolboxEncoder::UpdateFrameSize, encoder_,
 239                      frame_size));
 240       pool_frame_size_ = frame_size;
 241       pool_.reset();
 242       return nullptr;
 243     }
 244
 245     if (!pool_) {
 246       DVLOG(1) << "MaybeCreateFrame: No pixel buffer pool.";
 247       return nullptr;
 248     }
 249
 250     // Allocate a pixel buffer from the pool and return a wrapper VideoFrame.
 251     base::ScopedCFTypeRef<CVPixelBufferRef> buffer;
 252     auto status = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool_,
 253                                                      buffer.InitializeInto());
 254     if (status != kCVReturnSuccess) {
 255       DLOG(ERROR) << "CVPixelBufferPoolCreatePixelBuffer failed: " << status;
 256       return nullptr;
 257     }
 258
 259     DCHECK(buffer);
 260     return VideoFrame::WrapCVPixelBuffer(buffer, timestamp);
 261   }
 262
 263   void Update(const base::ScopedCFTypeRef<CVPixelBufferPoolRef>& pool,
 264               const gfx::Size& frame_size) {
 265     base::AutoLock auto_lock(lock_);
 266     pool_ = pool;
 267     pool_frame_size_ = frame_size;
 268   }
 269
 270  private:
 271   friend class base::RefCountedThreadSafe<VideoFrameFactoryImpl>;
 272   ~VideoFrameFactoryImpl() final {}
 273
 274   base::Lock lock_;
 275   base::ScopedCFTypeRef<CVPixelBufferPoolRef> pool_;
 276   gfx::Size pool_frame_size_;
 277
 278   // Weak back reference to the encoder and the cast envrionment so we can
 279   // message the encoder when the frame size changes.
 280   const base::WeakPtr<H264VideoToolboxEncoder> encoder_;
 281   const scoped_refptr<CastEnvironment> cast_environment_;
 282
 283   DISALLOW_COPY_AND_ASSIGN(VideoFrameFactoryImpl);
 284 };
 285
 286 class H264VideoToolboxEncoder::VideoFrameFactoryImpl::Proxy
 287     : public VideoFrameFactory {
 288  public:
 289   explicit Proxy(
 290       const scoped_refptr<VideoFrameFactoryImpl>& video_frame_factory)
 291       : video_frame_factory_(video_frame_factory) {
 292     DCHECK(video_frame_factory_);
 293   }
 294
 295   scoped_refptr<VideoFrame> MaybeCreateFrame(
 296       const gfx::Size& frame_size,
 297       base::TimeDelta timestamp) final {
 298     return video_frame_factory_->MaybeCreateFrame(frame_size, timestamp);
 299   }
 300
 301  private:
 302   ~Proxy() final {}
 303
 304   const scoped_refptr<VideoFrameFactoryImpl> video_frame_factory_;
 305
 306   DISALLOW_COPY_AND_ASSIGN(Proxy);
 307 };
 308
 309 // static
 310 bool H264VideoToolboxEncoder::IsSupported(
 311     const VideoSenderConfig& video_config) {
 312   return video_config.codec == CODEC_VIDEO_H264 && VideoToolboxGlue::Get();
 313 }
 314
 315 H264VideoToolboxEncoder::H264VideoToolboxEncoder(
 316     const scoped_refptr<CastEnvironment>& cast_environment,
 317     const VideoSenderConfig& video_config,
 318     const StatusChangeCallback& status_change_cb)
 319     : cast_environment_(cast_environment),
 320       videotoolbox_glue_(VideoToolboxGlue::Get()),
 321       video_config_(video_config),
 322       status_change_cb_(status_change_cb),
 323       last_frame_id_(kStartFrameId),
 324       encode_next_frame_as_keyframe_(false),
 325       power_suspended_(false),
 326       weak_factory_(this) {
 327   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 328   DCHECK(!status_change_cb_.is_null());
 329
 330   OperationalStatus operational_status =
 331       H264VideoToolboxEncoder::IsSupported(video_config)
 332           ? STATUS_INITIALIZED
 333           : STATUS_UNSUPPORTED_CODEC;
 334   cast_environment_->PostTask(
 335       CastEnvironment::MAIN, FROM_HERE,
 336       base::Bind(status_change_cb_, operational_status));
 337
 338   if (operational_status == STATUS_INITIALIZED) {
 339     // Create the shared video frame factory. It persists for the combined
 340     // lifetime of the encoder and all video frame factory proxies created by
 341     // |CreateVideoFrameFactory| that reference it.
 342     video_frame_factory_ =
 343         scoped_refptr<VideoFrameFactoryImpl>(new VideoFrameFactoryImpl(
 344             weak_factory_.GetWeakPtr(), cast_environment_));
 345
 346     // Register for power state changes.
 347     auto power_monitor = base::PowerMonitor::Get();
 348     if (power_monitor) {
 349       power_monitor->AddObserver(this);
 350       VLOG(1) << "Registered for power state changes.";
 351     } else {
 352       DLOG(WARNING) << "No power monitor. Process suspension will invalidate "
 353                        "the encoder.";
 354     }
 355   }
 356 }
 357
 358 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() {
 359   DestroyCompressionSession();
 360
 361   // If video_frame_factory_ is not null, the encoder registered for power state
 362   // changes in the ctor and it must now unregister.
 363   if (video_frame_factory_) {
 364     auto power_monitor = base::PowerMonitor::Get();
 365     if (power_monitor)
 366       power_monitor->RemoveObserver(this);
 367   }
 368 }
 369
 370 void H264VideoToolboxEncoder::ResetCompressionSession() {
 371   DCHECK(thread_checker_.CalledOnValidThread());
 372
 373   // Ignore reset requests while power suspended.
 374   if (power_suspended_)
 375     return;
 376
 377   // Notify that we're resetting the encoder.
 378   cast_environment_->PostTask(
 379       CastEnvironment::MAIN, FROM_HERE,
 380       base::Bind(status_change_cb_, STATUS_CODEC_REINIT_PENDING));
 381
 382   // Destroy the current session, if any.
 383   DestroyCompressionSession();
 384
 385   // On OS X, allow the hardware encoder. Don't require it, it does not support
 386   // all configurations (some of which are used for testing).
 387   base::ScopedCFTypeRef<CFDictionaryRef> encoder_spec;
 388 #if !defined(OS_IOS)
 389   encoder_spec = DictionaryWithKeyValue(
 390       videotoolbox_glue_
 391           ->kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder(),
 392       kCFBooleanTrue);
 393 #endif
 394
 395   // Force 420v so that clients can easily use these buffers as GPU textures.
 396   const int format[] = {
 397       CoreVideoGlue::kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange};
 398
 399   // Keep these attachment settings in-sync with those in ConfigureSession().
 400   CFTypeRef attachments_keys[] = {kCVImageBufferColorPrimariesKey,
 401                                   kCVImageBufferTransferFunctionKey,
 402                                   kCVImageBufferYCbCrMatrixKey};
 403   CFTypeRef attachments_values[] = {kCVImageBufferColorPrimaries_ITU_R_709_2,
 404                                     kCVImageBufferTransferFunction_ITU_R_709_2,
 405                                     kCVImageBufferYCbCrMatrix_ITU_R_709_2};
 406   CFTypeRef buffer_attributes_keys[] = {kCVPixelBufferPixelFormatTypeKey,
 407                                         kCVBufferPropagatedAttachmentsKey};
 408   CFTypeRef buffer_attributes_values[] = {
 409       ArrayWithIntegers(format, arraysize(format)).release(),
 410       DictionaryWithKeysAndValues(attachments_keys, attachments_values,
 411                                   arraysize(attachments_keys)).release()};
 412   const base::ScopedCFTypeRef<CFDictionaryRef> buffer_attributes =
 413       DictionaryWithKeysAndValues(buffer_attributes_keys,
 414                                   buffer_attributes_values,
 415                                   arraysize(buffer_attributes_keys));
 416   for (auto& v : buffer_attributes_values)
 417     CFRelease(v);
 418
 419   // Create the compression session.
 420
 421   // Note that the encoder object is given to the compression session as the
 422   // callback context using a raw pointer. The C API does not allow us to use a
 423   // smart pointer, nor is this encoder ref counted. However, this is still
 424   // safe, because we 1) we own the compression session and 2) we tear it down
 425   // safely. When destructing the encoder, the compression session is flushed
 426   // and invalidated. Internally, VideoToolbox will join all of its threads
 427   // before returning to the client. Therefore, when control returns to us, we
 428   // are guaranteed that the output callback will not execute again.
 429   OSStatus status = videotoolbox_glue_->VTCompressionSessionCreate(
 430       kCFAllocatorDefault, frame_size_.width(), frame_size_.height(),
 431       CoreMediaGlue::kCMVideoCodecType_H264, encoder_spec, buffer_attributes,
 432       nullptr /* compressedDataAllocator */,
 433       &H264VideoToolboxEncoder::CompressionCallback,
 434       reinterpret_cast<void*>(this), compression_session_.InitializeInto());
 435   if (status != noErr) {
 436     DLOG(ERROR) << " VTCompressionSessionCreate failed: " << status;
 437     // Notify that reinitialization has failed.
 438     cast_environment_->PostTask(
 439         CastEnvironment::MAIN, FROM_HERE,
 440         base::Bind(status_change_cb_, STATUS_CODEC_INIT_FAILED));
 441     return;
 442   }
 443
 444   // Configure the session (apply session properties based on the current state
 445   // of the encoder, experimental tuning and requirements).
 446   ConfigureCompressionSession();
 447
 448   // Update the video frame factory.
 449   base::ScopedCFTypeRef<CVPixelBufferPoolRef> pool(
 450       videotoolbox_glue_->VTCompressionSessionGetPixelBufferPool(
 451           compression_session_),
 452       base::scoped_policy::RETAIN);
 453   video_frame_factory_->Update(pool, frame_size_);
 454
 455   // Notify that reinitialization is done.
 456   cast_environment_->PostTask(
 457       CastEnvironment::MAIN, FROM_HERE,
 458       base::Bind(status_change_cb_, STATUS_INITIALIZED));
 459 }
 460
 461 void H264VideoToolboxEncoder::ConfigureCompressionSession() {
 462   SetSessionProperty(
 463       videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(),
 464       videotoolbox_glue_->kVTProfileLevel_H264_Main_AutoLevel());
 465   SetSessionProperty(videotoolbox_glue_->kVTCompressionPropertyKey_RealTime(),
 466                      true);
 467   SetSessionProperty(
 468       videotoolbox_glue_->kVTCompressionPropertyKey_AllowFrameReordering(),
 469       false);
 470   SetSessionProperty(
 471       videotoolbox_glue_->kVTCompressionPropertyKey_MaxKeyFrameInterval(), 240);
 472   SetSessionProperty(
 473       videotoolbox_glue_
 474           ->kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration(),
 475       240);
 476   // TODO(jfroy): implement better bitrate control
 477   //              https://crbug.com/425352
 478   SetSessionProperty(
 479       videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(),
 480       (video_config_.min_bitrate + video_config_.max_bitrate) / 2);
 481   SetSessionProperty(
 482       videotoolbox_glue_->kVTCompressionPropertyKey_ExpectedFrameRate(),
 483       video_config_.max_frame_rate);
 484   // Keep these attachment settings in-sync with those in Initialize().
 485   SetSessionProperty(
 486       videotoolbox_glue_->kVTCompressionPropertyKey_ColorPrimaries(),
 487       kCVImageBufferColorPrimaries_ITU_R_709_2);
 488   SetSessionProperty(
 489       videotoolbox_glue_->kVTCompressionPropertyKey_TransferFunction(),
 490       kCVImageBufferTransferFunction_ITU_R_709_2);
 491   SetSessionProperty(
 492       videotoolbox_glue_->kVTCompressionPropertyKey_YCbCrMatrix(),
 493       kCVImageBufferYCbCrMatrix_ITU_R_709_2);
 494   if (video_config_.max_number_of_video_buffers_used > 0) {
 495     SetSessionProperty(
 496         videotoolbox_glue_->kVTCompressionPropertyKey_MaxFrameDelayCount(),
 497         video_config_.max_number_of_video_buffers_used);
 498   }
 499 }
 500
 501 void H264VideoToolboxEncoder::DestroyCompressionSession() {
 502   DCHECK(thread_checker_.CalledOnValidThread());
 503
 504   // If the compression session exists, invalidate it. This blocks until all
 505   // pending output callbacks have returned and any internal threads have
 506   // joined, ensuring no output callback ever sees a dangling encoder pointer.
 507   //
 508   // Before destroying the compression session, the video frame factory's pool
 509   // is updated to null so that no thread will produce new video frames via the
 510   // factory until a new compression session is created. The current frame size
 511   // is passed to prevent the video frame factory from posting |UpdateFrameSize|
 512   // tasks. Indeed, |DestroyCompressionSession| is either called from
 513   // |ResetCompressionSession|, in which case a new pool and frame size will be
 514   // set, or from callsites that require that there be no compression session
 515   // (ex: the dtor).
 516   if (compression_session_) {
 517     video_frame_factory_->Update(
 518         base::ScopedCFTypeRef<CVPixelBufferPoolRef>(nullptr), frame_size_);
 519     videotoolbox_glue_->VTCompressionSessionInvalidate(compression_session_);
 520     compression_session_.reset();
 521   }
 522 }
 523
 524 bool H264VideoToolboxEncoder::EncodeVideoFrame(
 525     const scoped_refptr<media::VideoFrame>& video_frame,
 526     const base::TimeTicks& reference_time,
 527     const FrameEncodedCallback& frame_encoded_callback) {
 528   DCHECK(thread_checker_.CalledOnValidThread());
 529   DCHECK(!frame_encoded_callback.is_null());
 530
 531   // Reject empty video frames.
 532   const gfx::Size frame_size = video_frame->visible_rect().size();
 533   if (frame_size.IsEmpty()) {
 534     DVLOG(1) << "Rejecting empty video frame.";
 535     return false;
 536   }
 537
 538   // Handle frame size changes. This will reset the compression session.
 539   if (frame_size != frame_size_) {
 540     DVLOG(1) << "EncodeVideoFrame: Detected frame size change.";
 541     UpdateFrameSize(frame_size);
 542   }
 543
 544   // Need a compression session to continue.
 545   if (!compression_session_) {
 546     DLOG(ERROR) << "No compression session.";
 547     return false;
 548   }
 549
 550   // Wrap the VideoFrame in a CVPixelBuffer. In all cases, no data will be
 551   // copied. If the VideoFrame was created by this encoder's video frame
 552   // factory, then the returned CVPixelBuffer will have been obtained from the
 553   // compression session's pixel buffer pool. This will eliminate a copy of the
 554   // frame into memory visible by the hardware encoder. The VideoFrame's
 555   // lifetime is extended for the lifetime of the returned CVPixelBuffer.
 556   auto pixel_buffer = media::WrapVideoFrameInCVPixelBuffer(*video_frame);
 557   if (!pixel_buffer) {
 558     DLOG(ERROR) << "WrapVideoFrameInCVPixelBuffer failed.";
 559     return false;
 560   }
 561
 562   // Convert the frame timestamp to CMTime.
 563   auto timestamp_cm = CoreMediaGlue::CMTimeMake(
 564       (reference_time - base::TimeTicks()).InMicroseconds(), USEC_PER_SEC);
 565
 566   // Wrap information we'll need after the frame is encoded in a heap object.
 567   // We'll get the pointer back from the VideoToolbox completion callback.
 568   scoped_ptr<InProgressFrameEncode> request(new InProgressFrameEncode(
 569       TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency),
 570       reference_time, frame_encoded_callback));
 571
 572   // Build a suitable frame properties dictionary for keyframes.
 573   base::ScopedCFTypeRef<CFDictionaryRef> frame_props;
 574   if (encode_next_frame_as_keyframe_) {
 575     frame_props = DictionaryWithKeyValue(
 576         videotoolbox_glue_->kVTEncodeFrameOptionKey_ForceKeyFrame(),
 577         kCFBooleanTrue);
 578     encode_next_frame_as_keyframe_ = false;
 579   }
 580
 581   // Submit the frame to the compression session. The function returns as soon
 582   // as the frame has been enqueued.
 583   OSStatus status = videotoolbox_glue_->VTCompressionSessionEncodeFrame(
 584       compression_session_, pixel_buffer, timestamp_cm,
 585       CoreMediaGlue::CMTime{0, 0, 0, 0}, frame_props,
 586       reinterpret_cast<void*>(request.release()), nullptr);
 587   if (status != noErr) {
 588     DLOG(ERROR) << " VTCompressionSessionEncodeFrame failed: " << status;
 589     return false;
 590   }
 591
 592   return true;
 593 }
 594
 595 void H264VideoToolboxEncoder::UpdateFrameSize(const gfx::Size& size_needed) {
 596   DCHECK(thread_checker_.CalledOnValidThread());
 597
 598   // Our video frame factory posts a task to update the frame size when its
 599   // cache of the frame size differs from what the client requested. To avoid
 600   // spurious encoder resets, check again here.
 601   if (size_needed == frame_size_) {
 602     DCHECK(compression_session_);
 603     return;
 604   }
 605
 606   VLOG(1) << "Resetting compression session (for frame size change from "
 607           << frame_size_.ToString() << " to " << size_needed.ToString() << ").";
 608
 609   // If there is an existing session, finish every pending frame.
 610   if (compression_session_) {
 611     EmitFrames();
 612   }
 613
 614   // Store the new frame size.
 615   frame_size_ = size_needed;
 616
 617   // Reset the compression session.
 618   ResetCompressionSession();
 619 }
 620
 621 void H264VideoToolboxEncoder::SetBitRate(int /*new_bit_rate*/) {
 622   DCHECK(thread_checker_.CalledOnValidThread());
 623   // VideoToolbox does not seem to support bitrate reconfiguration.
 624 }
 625
 626 void H264VideoToolboxEncoder::GenerateKeyFrame() {
 627   DCHECK(thread_checker_.CalledOnValidThread());
 628   encode_next_frame_as_keyframe_ = true;
 629 }
 630
 631 void H264VideoToolboxEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) {
 632   // Not supported by VideoToolbox in any meaningful manner.
 633 }
 634
 635 scoped_ptr<VideoFrameFactory>
 636 H264VideoToolboxEncoder::CreateVideoFrameFactory() {
 637   DCHECK(thread_checker_.CalledOnValidThread());
 638   return scoped_ptr<VideoFrameFactory>(
 639       new VideoFrameFactoryImpl::Proxy(video_frame_factory_));
 640 }
 641
 642 void H264VideoToolboxEncoder::EmitFrames() {
 643   DCHECK(thread_checker_.CalledOnValidThread());
 644   if (!compression_session_)
 645     return;
 646
 647   OSStatus status = videotoolbox_glue_->VTCompressionSessionCompleteFrames(
 648       compression_session_, CoreMediaGlue::CMTime{0, 0, 0, 0});
 649   if (status != noErr) {
 650     DLOG(ERROR) << " VTCompressionSessionCompleteFrames failed: " << status;
 651   }
 652 }
 653
 654 void H264VideoToolboxEncoder::OnSuspend() {
 655   VLOG(1)
 656       << "OnSuspend: Emitting all frames and destroying compression session.";
 657   EmitFrames();
 658   DestroyCompressionSession();
 659   power_suspended_ = true;
 660 }
 661
 662 void H264VideoToolboxEncoder::OnResume() {
 663   power_suspended_ = false;
 664
 665   // Reset the compression session only if the frame size is not zero (which
 666   // will obviously fail). It is possible for the frame size to be zero if no
 667   // frame was submitted for encoding or requested from the video frame factory
 668   // before suspension.
 669   if (!frame_size_.IsEmpty()) {
 670     VLOG(1) << "OnResume: Resetting compression session.";
 671     ResetCompressionSession();
 672   }
 673 }
 674
 675 bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key,
 676                                                  int32_t value) {
 677   base::ScopedCFTypeRef<CFNumberRef> cfvalue(
 678       CFNumberCreate(nullptr, kCFNumberSInt32Type, &value));
 679   return videotoolbox_glue_->VTSessionSetProperty(compression_session_, key,
 680                                                   cfvalue) == noErr;
 681 }
 682
 683 bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key, bool value) {
 684   CFBooleanRef cfvalue = (value) ? kCFBooleanTrue : kCFBooleanFalse;
 685   return videotoolbox_glue_->VTSessionSetProperty(compression_session_, key,
 686                                                   cfvalue) == noErr;
 687 }
 688
 689 bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key,
 690                                                  CFStringRef value) {
 691   return videotoolbox_glue_->VTSessionSetProperty(compression_session_, key,
 692                                                   value) == noErr;
 693 }
 694
 695 void H264VideoToolboxEncoder::CompressionCallback(void* encoder_opaque,
 696                                                   void* request_opaque,
 697                                                   OSStatus status,
 698                                                   VTEncodeInfoFlags info,
 699                                                   CMSampleBufferRef sbuf) {
 700   auto encoder = reinterpret_cast<H264VideoToolboxEncoder*>(encoder_opaque);
 701   const scoped_ptr<InProgressFrameEncode> request(
 702       reinterpret_cast<InProgressFrameEncode*>(request_opaque));
 703   bool keyframe = false;
 704   bool has_frame_data = false;
 705
 706   if (status != noErr) {
 707     DLOG(ERROR) << " encoding failed: " << status;
 708     encoder->cast_environment_->PostTask(
 709         CastEnvironment::MAIN, FROM_HERE,
 710         base::Bind(encoder->status_change_cb_, STATUS_CODEC_RUNTIME_ERROR));
 711   } else if ((info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped)) {
 712     DVLOG(2) << " frame dropped";
 713   } else {
 714     auto sample_attachments =
 715         static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(
 716             CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true),
 717             0));
 718
 719     // If the NotSync key is not present, it implies Sync, which indicates a
 720     // keyframe (at least I think, VT documentation is, erm, sparse). Could
 721     // alternatively use kCMSampleAttachmentKey_DependsOnOthers == false.
 722     keyframe = !CFDictionaryContainsKey(
 723                    sample_attachments,
 724                    CoreMediaGlue::kCMSampleAttachmentKey_NotSync());
 725     has_frame_data = true;
 726   }
 727
 728   // Increment the encoder-scoped frame id and assign the new value to this
 729   // frame. VideoToolbox calls the output callback serially, so this is safe.
 730   const uint32 frame_id = ++encoder->last_frame_id_;
 731
 732   scoped_ptr<SenderEncodedFrame> encoded_frame(new SenderEncodedFrame());
 733   encoded_frame->frame_id = frame_id;
 734   encoded_frame->reference_time = request->reference_time;
 735   encoded_frame->rtp_timestamp = request->rtp_timestamp;
 736   if (keyframe) {
 737     encoded_frame->dependency = EncodedFrame::KEY;
 738     encoded_frame->referenced_frame_id = frame_id;
 739   } else {
 740     encoded_frame->dependency = EncodedFrame::DEPENDENT;
 741     // H.264 supports complex frame reference schemes (multiple reference
 742     // frames, slice references, backward and forward references, etc). Cast
 743     // doesn't support the concept of forward-referencing frame dependencies or
 744     // multiple frame dependencies; so pretend that all frames are only
 745     // decodable after their immediately preceding frame is decoded. This will
 746     // ensure a Cast receiver only attempts to decode the frames sequentially
 747     // and in order. Furthermore, the encoder is configured to never use forward
 748     // references (see |kVTCompressionPropertyKey_AllowFrameReordering|). There
 749     // is no way to prevent multiple reference frames.
 750     encoded_frame->referenced_frame_id = frame_id - 1;
 751   }
 752
 753   if (has_frame_data)
 754     CopySampleBufferToAnnexBBuffer(sbuf, &encoded_frame->data, keyframe);
 755
 756   // TODO(miu): Compute and populate the |deadline_utilization| and
 757   // |lossy_utilization| performance metrics in |encoded_frame|.
 758
 759   encoder->cast_environment_->PostTask(
 760       CastEnvironment::MAIN, FROM_HERE,
 761       base::Bind(request->frame_encoded_callback,
 762                  base::Passed(&encoded_frame)));
 763 }
 764
 765 }  // namespace cast
 766 }  // namespace media