content/common/gpu/media/android_video_encode_accelerator.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/common/gpu/media/android_video_encode_accelerator.h"
   6
   7 #include <set>
   8
   9 #include "base/bind.h"
  10 #include "base/command_line.h"
  11 #include "base/logging.h"
  12 #include "base/message_loop/message_loop.h"
  13 #include "base/metrics/histogram.h"
  14 #include "content/common/gpu/gpu_channel.h"
  15 #include "content/public/common/content_switches.h"
  16 #include "gpu/command_buffer/service/gles2_cmd_decoder.h"
  17 #include "media/base/android/media_codec_bridge.h"
  18 #include "media/base/bitstream_buffer.h"
  19 #include "media/base/limits.h"
  20 #include "media/video/picture.h"
  21 #include "third_party/libyuv/include/libyuv/convert_from.h"
  22 #include "ui/gl/android/scoped_java_surface.h"
  23 #include "ui/gl/gl_bindings.h"
  24
  25 using media::MediaCodecBridge;
  26 using media::VideoCodecBridge;
  27 using media::VideoFrame;
  28
  29 namespace content {
  30
  31 // Limit default max video codec size for Android to avoid
  32 // HW codec initialization failure for resolution higher than 720p.
  33 // Default values are from Libjingle "jsepsessiondescription.cc".
  34 const int kMaxEncodeFrameWidth = 1280;
  35 const int kMaxEncodeFrameHeight = 720;
  36 const int kMaxFramerateNumerator = 30;
  37 const int kMaxFramerateDenominator = 1;
  38
  39 enum PixelFormat {
  40   // Subset of MediaCodecInfo.CodecCapabilities.
  41   COLOR_FORMAT_YUV420_PLANAR = 19,
  42   COLOR_FORMAT_YUV420_SEMIPLANAR = 21,
  43 };
  44
  45 // Helper macros for dealing with failure.  If |result| evaluates false, emit
  46 // |log| to DLOG(ERROR), register |error| with the client, and return.
  47 #define RETURN_ON_FAILURE(result, log, error)                  \
  48   do {                                                         \
  49     if (!(result)) {                                           \
  50       DLOG(ERROR) << log;                                      \
  51       if (client_ptr_factory_->GetWeakPtr()) {                 \
  52         client_ptr_factory_->GetWeakPtr()->NotifyError(error); \
  53         client_ptr_factory_.reset();                           \
  54       }                                                        \
  55       return;                                                  \
  56     }                                                          \
  57   } while (0)
  58
  59 // Because MediaCodec is thread-hostile (must be poked on a single thread) and
  60 // has no callback mechanism (b/11990118), we must drive it by polling for
  61 // complete frames (and available input buffers, when the codec is fully
  62 // saturated).  This function defines the polling delay.  The value used is an
  63 // arbitrary choice that trades off CPU utilization (spinning) against latency.
  64 // Mirrors android_video_decode_accelerator.cc::DecodePollDelay().
  65 static inline const base::TimeDelta EncodePollDelay() {
  66   // An alternative to this polling scheme could be to dedicate a new thread
  67   // (instead of using the ChildThread) to run the MediaCodec, and make that
  68   // thread use the timeout-based flavor of MediaCodec's dequeue methods when it
  69   // believes the codec should complete "soon" (e.g. waiting for an input
  70   // buffer, or waiting for a picture when it knows enough complete input
  71   // pictures have been fed to saturate any internal buffering).  This is
  72   // speculative and it's unclear that this would be a win (nor that there's a
  73   // reasonably device-agnostic way to fill in the "believes" above).
  74   return base::TimeDelta::FromMilliseconds(10);
  75 }
  76
  77 static inline const base::TimeDelta NoWaitTimeOut() {
  78   return base::TimeDelta::FromMicroseconds(0);
  79 }
  80
  81 static bool GetSupportedColorFormatForMime(const std::string& mime,
  82                                            PixelFormat* pixel_format) {
  83   if (mime.empty())
  84     return false;
  85
  86   std::set<int> formats = MediaCodecBridge::GetEncoderColorFormats(mime);
  87   if (formats.count(COLOR_FORMAT_YUV420_SEMIPLANAR) > 0)
  88     *pixel_format = COLOR_FORMAT_YUV420_SEMIPLANAR;
  89   else if (formats.count(COLOR_FORMAT_YUV420_PLANAR) > 0)
  90     *pixel_format = COLOR_FORMAT_YUV420_PLANAR;
  91   else
  92     return false;
  93
  94   return true;
  95 }
  96
  97 AndroidVideoEncodeAccelerator::AndroidVideoEncodeAccelerator()
  98     : num_buffers_at_codec_(0),
  99       num_output_buffers_(-1),
 100       output_buffers_capacity_(0),
 101       last_set_bitrate_(0) {}
 102
 103 AndroidVideoEncodeAccelerator::~AndroidVideoEncodeAccelerator() {
 104   DCHECK(thread_checker_.CalledOnValidThread());
 105 }
 106
 107 media::VideoEncodeAccelerator::SupportedProfiles
 108 AndroidVideoEncodeAccelerator::GetSupportedProfiles() {
 109   SupportedProfiles profiles;
 110
 111 #if defined(ENABLE_WEBRTC)
 112   const base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess();
 113   if (cmd_line->HasSwitch(switches::kDisableWebRtcHWEncoding))
 114     return profiles;
 115 #endif
 116
 117   const struct {
 118       const media::VideoCodec codec;
 119       const media::VideoCodecProfile profile;
 120   } kSupportedCodecs[] = {
 121       { media::kCodecVP8, media::VP8PROFILE_ANY },
 122       { media::kCodecH264, media::H264PROFILE_BASELINE },
 123       { media::kCodecH264, media::H264PROFILE_MAIN }
 124   };
 125
 126   for (const auto& supported_codec : kSupportedCodecs) {
 127     if (VideoCodecBridge::IsKnownUnaccelerated(supported_codec.codec,
 128                                                media::MEDIA_CODEC_ENCODER)) {
 129       continue;
 130     }
 131
 132     SupportedProfile profile;
 133     profile.profile = supported_codec.profile;
 134     // It would be nice if MediaCodec exposes the maximum capabilities of
 135     // the encoder. Hard-code some reasonable defaults as workaround.
 136     profile.max_resolution.SetSize(kMaxEncodeFrameWidth,
 137                                    kMaxEncodeFrameHeight);
 138     profile.max_framerate_numerator = kMaxFramerateNumerator;
 139     profile.max_framerate_denominator = kMaxFramerateDenominator;
 140     profiles.push_back(profile);
 141   }
 142   return profiles;
 143 }
 144
 145 bool AndroidVideoEncodeAccelerator::Initialize(
 146     media::VideoPixelFormat format,
 147     const gfx::Size& input_visible_size,
 148     media::VideoCodecProfile output_profile,
 149     uint32 initial_bitrate,
 150     Client* client) {
 151   DVLOG(3) << __PRETTY_FUNCTION__ << " format: " << format
 152            << ", input_visible_size: " << input_visible_size.ToString()
 153            << ", output_profile: " << output_profile
 154            << ", initial_bitrate: " << initial_bitrate;
 155   DCHECK(!media_codec_);
 156   DCHECK(thread_checker_.CalledOnValidThread());
 157
 158   client_ptr_factory_.reset(new base::WeakPtrFactory<Client>(client));
 159
 160   if (!(media::MediaCodecBridge::SupportsSetParameters() &&
 161         format == media::PIXEL_FORMAT_I420)) {
 162     DLOG(ERROR) << "Unexpected combo: " << format << ", " << output_profile;
 163     return false;
 164   }
 165
 166   std::string mime_type;
 167   media::VideoCodec codec;
 168   if (output_profile == media::VP8PROFILE_ANY) {
 169     codec = media::kCodecVP8;
 170     mime_type = "video/x-vnd.on2.vp8";
 171   } else if (output_profile == media::H264PROFILE_BASELINE ||
 172              output_profile == media::H264PROFILE_MAIN) {
 173     codec = media::kCodecH264;
 174     mime_type = "video/avc";
 175   } else {
 176     return false;
 177   }
 178
 179   last_set_bitrate_ = initial_bitrate;
 180
 181   // Only consider using MediaCodec if it's likely backed by hardware.
 182   if (media::VideoCodecBridge::IsKnownUnaccelerated(
 183           codec, media::MEDIA_CODEC_ENCODER)) {
 184     DLOG(ERROR) << "No HW support";
 185     return false;
 186   }
 187
 188   PixelFormat pixel_format = COLOR_FORMAT_YUV420_SEMIPLANAR;
 189   if (!GetSupportedColorFormatForMime(mime_type, &pixel_format)) {
 190     DLOG(ERROR) << "No color format support.";
 191     return false;
 192   }
 193   media_codec_.reset(media::VideoCodecBridge::CreateEncoder(codec,
 194                                                             input_visible_size,
 195                                                             initial_bitrate,
 196                                                             INITIAL_FRAMERATE,
 197                                                             IFRAME_INTERVAL,
 198                                                             pixel_format));
 199
 200   if (!media_codec_) {
 201     DLOG(ERROR) << "Failed to create/start the codec: "
 202                 << input_visible_size.ToString();
 203     return false;
 204   }
 205
 206   num_output_buffers_ = media_codec_->GetOutputBuffersCount();
 207   output_buffers_capacity_ = media_codec_->GetOutputBuffersCapacity();
 208   base::MessageLoop::current()->PostTask(
 209       FROM_HERE,
 210       base::Bind(&VideoEncodeAccelerator::Client::RequireBitstreamBuffers,
 211                  client_ptr_factory_->GetWeakPtr(),
 212                  num_output_buffers_,
 213                  input_visible_size,
 214                  output_buffers_capacity_));
 215   return true;
 216 }
 217
 218 void AndroidVideoEncodeAccelerator::MaybeStartIOTimer() {
 219   if (!io_timer_.IsRunning() &&
 220       (num_buffers_at_codec_ > 0 || !pending_frames_.empty())) {
 221     io_timer_.Start(FROM_HERE,
 222                     EncodePollDelay(),
 223                     this,
 224                     &AndroidVideoEncodeAccelerator::DoIOTask);
 225   }
 226 }
 227
 228 void AndroidVideoEncodeAccelerator::MaybeStopIOTimer() {
 229   if (io_timer_.IsRunning() &&
 230       (num_buffers_at_codec_ == 0 && pending_frames_.empty())) {
 231     io_timer_.Stop();
 232   }
 233 }
 234
 235 void AndroidVideoEncodeAccelerator::Encode(
 236     const scoped_refptr<VideoFrame>& frame,
 237     bool force_keyframe) {
 238   DVLOG(3) << __PRETTY_FUNCTION__ << ": " << force_keyframe;
 239   DCHECK(thread_checker_.CalledOnValidThread());
 240   RETURN_ON_FAILURE(frame->format() == media::PIXEL_FORMAT_I420,
 241                     "Unexpected format", kInvalidArgumentError);
 242
 243   // MediaCodec doesn't have a way to specify stride for non-Packed formats, so
 244   // we insist on being called with packed frames and no cropping :(
 245   RETURN_ON_FAILURE(frame->row_bytes(VideoFrame::kYPlane) ==
 246                             frame->stride(VideoFrame::kYPlane) &&
 247                         frame->row_bytes(VideoFrame::kUPlane) ==
 248                             frame->stride(VideoFrame::kUPlane) &&
 249                         frame->row_bytes(VideoFrame::kVPlane) ==
 250                             frame->stride(VideoFrame::kVPlane) &&
 251                         frame->coded_size() == frame->visible_rect().size(),
 252                     "Non-packed frame, or visible_rect != coded_size",
 253                     kInvalidArgumentError);
 254
 255   pending_frames_.push(
 256       base::MakeTuple(frame, force_keyframe, base::Time::Now()));
 257   DoIOTask();
 258 }
 259
 260 void AndroidVideoEncodeAccelerator::UseOutputBitstreamBuffer(
 261     const media::BitstreamBuffer& buffer) {
 262   DVLOG(3) << __PRETTY_FUNCTION__ << ": bitstream_buffer_id=" << buffer.id();
 263   DCHECK(thread_checker_.CalledOnValidThread());
 264   RETURN_ON_FAILURE(buffer.size() >= media_codec_->GetOutputBuffersCapacity(),
 265                     "Output buffers too small!",
 266                     kInvalidArgumentError);
 267   available_bitstream_buffers_.push_back(buffer);
 268   DoIOTask();
 269 }
 270
 271 void AndroidVideoEncodeAccelerator::RequestEncodingParametersChange(
 272     uint32 bitrate,
 273     uint32 framerate) {
 274   DVLOG(3) << __PRETTY_FUNCTION__ << ": bitrate: " << bitrate
 275            << ", framerate: " << framerate;
 276   DCHECK(thread_checker_.CalledOnValidThread());
 277   if (bitrate != last_set_bitrate_) {
 278     last_set_bitrate_ = bitrate;
 279     media_codec_->SetVideoBitrate(bitrate);
 280   }
 281   // Note: Android's MediaCodec doesn't allow mid-stream adjustments to
 282   // framerate, so we ignore that here.  This is OK because Android only uses
 283   // the framerate value from MediaFormat during configure() as a proxy for
 284   // bitrate, and we set that explicitly.
 285 }
 286
 287 void AndroidVideoEncodeAccelerator::Destroy() {
 288   DVLOG(3) << __PRETTY_FUNCTION__;
 289   DCHECK(thread_checker_.CalledOnValidThread());
 290   client_ptr_factory_.reset();
 291   if (media_codec_) {
 292     if (io_timer_.IsRunning())
 293       io_timer_.Stop();
 294     media_codec_->Stop();
 295   }
 296   delete this;
 297 }
 298
 299 void AndroidVideoEncodeAccelerator::DoIOTask() {
 300   QueueInput();
 301   DequeueOutput();
 302   MaybeStartIOTimer();
 303   MaybeStopIOTimer();
 304 }
 305
 306 void AndroidVideoEncodeAccelerator::QueueInput() {
 307   if (!client_ptr_factory_->GetWeakPtr() || pending_frames_.empty())
 308     return;
 309
 310   int input_buf_index = 0;
 311   media::MediaCodecStatus status =
 312       media_codec_->DequeueInputBuffer(NoWaitTimeOut(), &input_buf_index);
 313   if (status != media::MEDIA_CODEC_OK) {
 314     DCHECK(status == media::MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER ||
 315            status == media::MEDIA_CODEC_ERROR);
 316     RETURN_ON_FAILURE(status != media::MEDIA_CODEC_ERROR,
 317                       "MediaCodec error",
 318                       kPlatformFailureError);
 319     return;
 320   }
 321
 322   const PendingFrames::value_type& input = pending_frames_.front();
 323   bool is_key_frame = base::get<1>(input);
 324   if (is_key_frame) {
 325     // Ideally MediaCodec would honor BUFFER_FLAG_SYNC_FRAME so we could
 326     // indicate this in the QueueInputBuffer() call below and guarantee _this_
 327     // frame be encoded as a key frame, but sadly that flag is ignored.
 328     // Instead, we request a key frame "soon".
 329     media_codec_->RequestKeyFrameSoon();
 330   }
 331   scoped_refptr<VideoFrame> frame = base::get<0>(input);
 332
 333   uint8* buffer = NULL;
 334   size_t capacity = 0;
 335   media_codec_->GetInputBuffer(input_buf_index, &buffer, &capacity);
 336
 337   size_t queued_size =
 338       VideoFrame::AllocationSize(media::PIXEL_FORMAT_I420, frame->coded_size());
 339   RETURN_ON_FAILURE(capacity >= queued_size,
 340                     "Failed to get input buffer: " << input_buf_index,
 341                     kPlatformFailureError);
 342
 343   uint8* dst_y = buffer;
 344   int dst_stride_y = frame->stride(VideoFrame::kYPlane);
 345   uint8* dst_uv = buffer + frame->stride(VideoFrame::kYPlane) *
 346                                frame->rows(VideoFrame::kYPlane);
 347   int dst_stride_uv = frame->stride(VideoFrame::kUPlane) * 2;
 348   // Why NV12?  Because COLOR_FORMAT_YUV420_SEMIPLANAR.  See comment at other
 349   // mention of that constant.
 350   bool converted = !libyuv::I420ToNV12(frame->data(VideoFrame::kYPlane),
 351                                        frame->stride(VideoFrame::kYPlane),
 352                                        frame->data(VideoFrame::kUPlane),
 353                                        frame->stride(VideoFrame::kUPlane),
 354                                        frame->data(VideoFrame::kVPlane),
 355                                        frame->stride(VideoFrame::kVPlane),
 356                                        dst_y,
 357                                        dst_stride_y,
 358                                        dst_uv,
 359                                        dst_stride_uv,
 360                                        frame->coded_size().width(),
 361                                        frame->coded_size().height());
 362   RETURN_ON_FAILURE(converted, "Failed to I420ToNV12!", kPlatformFailureError);
 363
 364   fake_input_timestamp_ += base::TimeDelta::FromMicroseconds(1);
 365   status = media_codec_->QueueInputBuffer(
 366       input_buf_index, NULL, queued_size, fake_input_timestamp_);
 367   UMA_HISTOGRAM_TIMES("Media.AVEA.InputQueueTime",
 368                       base::Time::Now() - base::get<2>(input));
 369   RETURN_ON_FAILURE(status == media::MEDIA_CODEC_OK,
 370                     "Failed to QueueInputBuffer: " << status,
 371                     kPlatformFailureError);
 372   ++num_buffers_at_codec_;
 373   pending_frames_.pop();
 374 }
 375
 376 bool AndroidVideoEncodeAccelerator::DoOutputBuffersSuffice() {
 377   // If this returns false ever, then the VEA::Client interface will need to
 378   // grow a DismissBitstreamBuffer() call, and VEA::Client impls will have to be
 379   // prepared to field multiple requests to RequireBitstreamBuffers().
 380   int count = media_codec_->GetOutputBuffersCount();
 381   size_t capacity = media_codec_->GetOutputBuffersCapacity();
 382   bool ret = count <= num_output_buffers_ &&
 383       capacity <= output_buffers_capacity_;
 384   LOG_IF(ERROR, !ret) << "Need more/bigger buffers; before: "
 385                       << num_output_buffers_ << "x" << output_buffers_capacity_
 386                       << ", now: " << count << "x" << capacity;
 387   UMA_HISTOGRAM_BOOLEAN("Media.AVEA.OutputBuffersSuffice", ret);
 388   return ret;
 389 }
 390
 391 void AndroidVideoEncodeAccelerator::DequeueOutput() {
 392   if (!client_ptr_factory_->GetWeakPtr() ||
 393       available_bitstream_buffers_.empty() || num_buffers_at_codec_ == 0) {
 394     return;
 395   }
 396
 397   int32 buf_index = 0;
 398   size_t offset = 0;
 399   size_t size = 0;
 400   bool key_frame = false;
 401   do {
 402     media::MediaCodecStatus status = media_codec_->DequeueOutputBuffer(
 403         NoWaitTimeOut(), &buf_index, &offset, &size, NULL, NULL, &key_frame);
 404     switch (status) {
 405       case media::MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER:
 406         return;
 407
 408       case media::MEDIA_CODEC_ERROR:
 409         RETURN_ON_FAILURE(false, "Codec error", kPlatformFailureError);
 410         // Unreachable because of previous statement, but included for clarity.
 411         return;
 412
 413       case media::MEDIA_CODEC_OUTPUT_FORMAT_CHANGED:  // Fall-through.
 414       case media::MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED:
 415         RETURN_ON_FAILURE(DoOutputBuffersSuffice(),
 416                           "Bitstream now requires more/larger buffers",
 417                           kPlatformFailureError);
 418         break;
 419
 420       case media::MEDIA_CODEC_OK:
 421         DCHECK_GE(buf_index, 0);
 422         break;
 423
 424       default:
 425         NOTREACHED();
 426         break;
 427     }
 428   } while (buf_index < 0);
 429
 430   media::BitstreamBuffer bitstream_buffer = available_bitstream_buffers_.back();
 431   available_bitstream_buffers_.pop_back();
 432   scoped_ptr<base::SharedMemory> shm(
 433       new base::SharedMemory(bitstream_buffer.handle(), false));
 434   RETURN_ON_FAILURE(shm->Map(bitstream_buffer.size()),
 435                     "Failed to map SHM",
 436                     kPlatformFailureError);
 437   RETURN_ON_FAILURE(size <= shm->mapped_size(),
 438                     "Encoded buffer too large: " << size << ">"
 439                                                  << shm->mapped_size(),
 440                     kPlatformFailureError);
 441
 442   media_codec_->CopyFromOutputBuffer(buf_index, offset, shm->memory(), size);
 443   media_codec_->ReleaseOutputBuffer(buf_index, false);
 444   --num_buffers_at_codec_;
 445
 446   UMA_HISTOGRAM_COUNTS_10000("Media.AVEA.EncodedBufferSizeKB", size / 1024);
 447   base::MessageLoop::current()->PostTask(
 448       FROM_HERE,
 449       base::Bind(&VideoEncodeAccelerator::Client::BitstreamBufferReady,
 450                  client_ptr_factory_->GetWeakPtr(),
 451                  bitstream_buffer.id(),
 452                  size,
 453                  key_frame));
 454 }
 455
 456 }  // namespace content