1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/common/gpu/media/android_video_encode_accelerator.h"
10 #include "base/command_line.h"
11 #include "base/logging.h"
12 #include "base/message_loop/message_loop.h"
13 #include "base/metrics/histogram.h"
14 #include "content/common/gpu/gpu_channel.h"
15 #include "content/public/common/content_switches.h"
16 #include "gpu/command_buffer/service/gles2_cmd_decoder.h"
17 #include "media/base/android/media_codec_bridge.h"
18 #include "media/base/bitstream_buffer.h"
19 #include "media/base/limits.h"
20 #include "media/video/picture.h"
21 #include "third_party/libyuv/include/libyuv/convert_from.h"
22 #include "ui/gl/android/scoped_java_surface.h"
23 #include "ui/gl/gl_bindings.h"
25 using media::MediaCodecBridge
;
26 using media::VideoCodecBridge
;
27 using media::VideoFrame
;
31 // Limit default max video codec size for Android to avoid
32 // HW codec initialization failure for resolution higher than 720p.
33 // Default values are from Libjingle "jsepsessiondescription.cc".
34 const int kMaxEncodeFrameWidth
= 1280;
35 const int kMaxEncodeFrameHeight
= 720;
36 const int kMaxFramerateNumerator
= 30;
37 const int kMaxFramerateDenominator
= 1;
40 // Subset of MediaCodecInfo.CodecCapabilities.
41 COLOR_FORMAT_YUV420_PLANAR
= 19,
42 COLOR_FORMAT_YUV420_SEMIPLANAR
= 21,
45 // Helper macros for dealing with failure. If |result| evaluates false, emit
46 // |log| to DLOG(ERROR), register |error| with the client, and return.
47 #define RETURN_ON_FAILURE(result, log, error) \
51 if (client_ptr_factory_->GetWeakPtr()) { \
52 client_ptr_factory_->GetWeakPtr()->NotifyError(error); \
53 client_ptr_factory_.reset(); \
59 // Because MediaCodec is thread-hostile (must be poked on a single thread) and
60 // has no callback mechanism (b/11990118), we must drive it by polling for
61 // complete frames (and available input buffers, when the codec is fully
62 // saturated). This function defines the polling delay. The value used is an
63 // arbitrary choice that trades off CPU utilization (spinning) against latency.
64 // Mirrors android_video_decode_accelerator.cc::DecodePollDelay().
65 static inline const base::TimeDelta
EncodePollDelay() {
66 // An alternative to this polling scheme could be to dedicate a new thread
67 // (instead of using the ChildThread) to run the MediaCodec, and make that
68 // thread use the timeout-based flavor of MediaCodec's dequeue methods when it
69 // believes the codec should complete "soon" (e.g. waiting for an input
70 // buffer, or waiting for a picture when it knows enough complete input
71 // pictures have been fed to saturate any internal buffering). This is
72 // speculative and it's unclear that this would be a win (nor that there's a
73 // reasonably device-agnostic way to fill in the "believes" above).
74 return base::TimeDelta::FromMilliseconds(10);
77 static inline const base::TimeDelta
NoWaitTimeOut() {
78 return base::TimeDelta::FromMicroseconds(0);
81 static bool GetSupportedColorFormatForMime(const std::string
& mime
,
82 PixelFormat
* pixel_format
) {
86 std::set
<int> formats
= MediaCodecBridge::GetEncoderColorFormats(mime
);
87 if (formats
.count(COLOR_FORMAT_YUV420_SEMIPLANAR
) > 0)
88 *pixel_format
= COLOR_FORMAT_YUV420_SEMIPLANAR
;
89 else if (formats
.count(COLOR_FORMAT_YUV420_PLANAR
) > 0)
90 *pixel_format
= COLOR_FORMAT_YUV420_PLANAR
;
97 AndroidVideoEncodeAccelerator::AndroidVideoEncodeAccelerator()
98 : num_buffers_at_codec_(0),
99 num_output_buffers_(-1),
100 output_buffers_capacity_(0),
101 last_set_bitrate_(0) {}
103 AndroidVideoEncodeAccelerator::~AndroidVideoEncodeAccelerator() {
104 DCHECK(thread_checker_
.CalledOnValidThread());
107 std::vector
<media::VideoEncodeAccelerator::SupportedProfile
>
108 AndroidVideoEncodeAccelerator::GetSupportedProfiles() {
109 std::vector
<SupportedProfile
> profiles
;
111 #if defined(ENABLE_WEBRTC)
112 const base::CommandLine
* cmd_line
= base::CommandLine::ForCurrentProcess();
113 if (cmd_line
->HasSwitch(switches::kDisableWebRtcHWEncoding
))
118 const media::VideoCodec codec
;
119 const media::VideoCodecProfile profile
;
120 } kSupportedCodecs
[] = {
121 { media::kCodecVP8
, media::VP8PROFILE_ANY
},
122 { media::kCodecH264
, media::H264PROFILE_BASELINE
},
123 { media::kCodecH264
, media::H264PROFILE_MAIN
}
126 for (const auto& supported_codec
: kSupportedCodecs
) {
127 if (VideoCodecBridge::IsKnownUnaccelerated(supported_codec
.codec
,
128 media::MEDIA_CODEC_ENCODER
)) {
132 SupportedProfile profile
;
133 profile
.profile
= supported_codec
.profile
;
134 // It would be nice if MediaCodec exposes the maximum capabilities of
135 // the encoder. Hard-code some reasonable defaults as workaround.
136 profile
.max_resolution
.SetSize(kMaxEncodeFrameWidth
,
137 kMaxEncodeFrameHeight
);
138 profile
.max_framerate_numerator
= kMaxFramerateNumerator
;
139 profile
.max_framerate_denominator
= kMaxFramerateDenominator
;
140 profiles
.push_back(profile
);
145 bool AndroidVideoEncodeAccelerator::Initialize(
146 VideoFrame::Format format
,
147 const gfx::Size
& input_visible_size
,
148 media::VideoCodecProfile output_profile
,
149 uint32 initial_bitrate
,
151 DVLOG(3) << __PRETTY_FUNCTION__
<< " format: " << format
152 << ", input_visible_size: " << input_visible_size
.ToString()
153 << ", output_profile: " << output_profile
154 << ", initial_bitrate: " << initial_bitrate
;
155 DCHECK(!media_codec_
);
156 DCHECK(thread_checker_
.CalledOnValidThread());
158 client_ptr_factory_
.reset(new base::WeakPtrFactory
<Client
>(client
));
160 if (!(media::MediaCodecBridge::SupportsSetParameters() &&
161 format
== VideoFrame::I420
)) {
162 DLOG(ERROR
) << "Unexpected combo: " << format
<< ", " << output_profile
;
166 std::string mime_type
;
167 media::VideoCodec codec
;
168 if (output_profile
== media::VP8PROFILE_ANY
) {
169 codec
= media::kCodecVP8
;
170 mime_type
= "video/x-vnd.on2.vp8";
171 } else if (output_profile
== media::H264PROFILE_BASELINE
||
172 output_profile
== media::H264PROFILE_MAIN
) {
173 codec
= media::kCodecH264
;
174 mime_type
= "video/avc";
179 last_set_bitrate_
= initial_bitrate
;
181 // Only consider using MediaCodec if it's likely backed by hardware.
182 if (media::VideoCodecBridge::IsKnownUnaccelerated(
183 codec
, media::MEDIA_CODEC_ENCODER
)) {
184 DLOG(ERROR
) << "No HW support";
188 PixelFormat pixel_format
= COLOR_FORMAT_YUV420_SEMIPLANAR
;
189 if (!GetSupportedColorFormatForMime(mime_type
, &pixel_format
)) {
190 DLOG(ERROR
) << "No color format support.";
193 media_codec_
.reset(media::VideoCodecBridge::CreateEncoder(codec
,
201 DLOG(ERROR
) << "Failed to create/start the codec: "
202 << input_visible_size
.ToString();
206 num_output_buffers_
= media_codec_
->GetOutputBuffersCount();
207 output_buffers_capacity_
= media_codec_
->GetOutputBuffersCapacity();
208 base::MessageLoop::current()->PostTask(
210 base::Bind(&VideoEncodeAccelerator::Client::RequireBitstreamBuffers
,
211 client_ptr_factory_
->GetWeakPtr(),
214 output_buffers_capacity_
));
218 void AndroidVideoEncodeAccelerator::MaybeStartIOTimer() {
219 if (!io_timer_
.IsRunning() &&
220 (num_buffers_at_codec_
> 0 || !pending_frames_
.empty())) {
221 io_timer_
.Start(FROM_HERE
,
224 &AndroidVideoEncodeAccelerator::DoIOTask
);
228 void AndroidVideoEncodeAccelerator::MaybeStopIOTimer() {
229 if (io_timer_
.IsRunning() &&
230 (num_buffers_at_codec_
== 0 && pending_frames_
.empty())) {
235 void AndroidVideoEncodeAccelerator::Encode(
236 const scoped_refptr
<VideoFrame
>& frame
,
237 bool force_keyframe
) {
238 DVLOG(3) << __PRETTY_FUNCTION__
<< ": " << force_keyframe
;
239 DCHECK(thread_checker_
.CalledOnValidThread());
240 RETURN_ON_FAILURE(frame
->format() == VideoFrame::I420
,
242 kInvalidArgumentError
);
244 // MediaCodec doesn't have a way to specify stride for non-Packed formats, so
245 // we insist on being called with packed frames and no cropping :(
246 RETURN_ON_FAILURE(frame
->row_bytes(VideoFrame::kYPlane
) ==
247 frame
->stride(VideoFrame::kYPlane
) &&
248 frame
->row_bytes(VideoFrame::kUPlane
) ==
249 frame
->stride(VideoFrame::kUPlane
) &&
250 frame
->row_bytes(VideoFrame::kVPlane
) ==
251 frame
->stride(VideoFrame::kVPlane
) &&
252 frame
->coded_size() == frame
->visible_rect().size(),
253 "Non-packed frame, or visible_rect != coded_size",
254 kInvalidArgumentError
);
256 pending_frames_
.push(MakeTuple(frame
, force_keyframe
, base::Time::Now()));
260 void AndroidVideoEncodeAccelerator::UseOutputBitstreamBuffer(
261 const media::BitstreamBuffer
& buffer
) {
262 DVLOG(3) << __PRETTY_FUNCTION__
<< ": bitstream_buffer_id=" << buffer
.id();
263 DCHECK(thread_checker_
.CalledOnValidThread());
264 RETURN_ON_FAILURE(buffer
.size() >= media_codec_
->GetOutputBuffersCapacity(),
265 "Output buffers too small!",
266 kInvalidArgumentError
);
267 available_bitstream_buffers_
.push_back(buffer
);
271 void AndroidVideoEncodeAccelerator::RequestEncodingParametersChange(
274 DVLOG(3) << __PRETTY_FUNCTION__
<< ": bitrate: " << bitrate
275 << ", framerate: " << framerate
;
276 DCHECK(thread_checker_
.CalledOnValidThread());
277 if (bitrate
!= last_set_bitrate_
) {
278 last_set_bitrate_
= bitrate
;
279 media_codec_
->SetVideoBitrate(bitrate
);
281 // Note: Android's MediaCodec doesn't allow mid-stream adjustments to
282 // framerate, so we ignore that here. This is OK because Android only uses
283 // the framerate value from MediaFormat during configure() as a proxy for
284 // bitrate, and we set that explicitly.
287 void AndroidVideoEncodeAccelerator::Destroy() {
288 DVLOG(3) << __PRETTY_FUNCTION__
;
289 DCHECK(thread_checker_
.CalledOnValidThread());
290 client_ptr_factory_
.reset();
292 if (io_timer_
.IsRunning())
294 media_codec_
->Stop();
299 void AndroidVideoEncodeAccelerator::DoIOTask() {
306 void AndroidVideoEncodeAccelerator::QueueInput() {
307 if (!client_ptr_factory_
->GetWeakPtr() || pending_frames_
.empty())
310 int input_buf_index
= 0;
311 media::MediaCodecStatus status
=
312 media_codec_
->DequeueInputBuffer(NoWaitTimeOut(), &input_buf_index
);
313 if (status
!= media::MEDIA_CODEC_OK
) {
314 DCHECK(status
== media::MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER
||
315 status
== media::MEDIA_CODEC_ERROR
);
316 RETURN_ON_FAILURE(status
!= media::MEDIA_CODEC_ERROR
,
318 kPlatformFailureError
);
322 const PendingFrames::value_type
& input
= pending_frames_
.front();
323 bool is_key_frame
= get
<1>(input
);
325 // Ideally MediaCodec would honor BUFFER_FLAG_SYNC_FRAME so we could
326 // indicate this in the QueueInputBuffer() call below and guarantee _this_
327 // frame be encoded as a key frame, but sadly that flag is ignored.
328 // Instead, we request a key frame "soon".
329 media_codec_
->RequestKeyFrameSoon();
331 scoped_refptr
<VideoFrame
> frame
= get
<0>(input
);
333 uint8
* buffer
= NULL
;
335 media_codec_
->GetInputBuffer(input_buf_index
, &buffer
, &capacity
);
338 VideoFrame::AllocationSize(VideoFrame::I420
, frame
->coded_size());
339 RETURN_ON_FAILURE(capacity
>= queued_size
,
340 "Failed to get input buffer: " << input_buf_index
,
341 kPlatformFailureError
);
343 uint8
* dst_y
= buffer
;
344 int dst_stride_y
= frame
->stride(VideoFrame::kYPlane
);
345 uint8
* dst_uv
= buffer
+ frame
->stride(VideoFrame::kYPlane
) *
346 frame
->rows(VideoFrame::kYPlane
);
347 int dst_stride_uv
= frame
->stride(VideoFrame::kUPlane
) * 2;
348 // Why NV12? Because COLOR_FORMAT_YUV420_SEMIPLANAR. See comment at other
349 // mention of that constant.
350 bool converted
= !libyuv::I420ToNV12(frame
->data(VideoFrame::kYPlane
),
351 frame
->stride(VideoFrame::kYPlane
),
352 frame
->data(VideoFrame::kUPlane
),
353 frame
->stride(VideoFrame::kUPlane
),
354 frame
->data(VideoFrame::kVPlane
),
355 frame
->stride(VideoFrame::kVPlane
),
360 frame
->coded_size().width(),
361 frame
->coded_size().height());
362 RETURN_ON_FAILURE(converted
, "Failed to I420ToNV12!", kPlatformFailureError
);
364 fake_input_timestamp_
+= base::TimeDelta::FromMicroseconds(1);
365 status
= media_codec_
->QueueInputBuffer(
366 input_buf_index
, NULL
, queued_size
, fake_input_timestamp_
);
367 UMA_HISTOGRAM_TIMES("Media.AVEA.InputQueueTime",
368 base::Time::Now() - get
<2>(input
));
369 RETURN_ON_FAILURE(status
== media::MEDIA_CODEC_OK
,
370 "Failed to QueueInputBuffer: " << status
,
371 kPlatformFailureError
);
372 ++num_buffers_at_codec_
;
373 pending_frames_
.pop();
376 bool AndroidVideoEncodeAccelerator::DoOutputBuffersSuffice() {
377 // If this returns false ever, then the VEA::Client interface will need to
378 // grow a DismissBitstreamBuffer() call, and VEA::Client impls will have to be
379 // prepared to field multiple requests to RequireBitstreamBuffers().
380 int count
= media_codec_
->GetOutputBuffersCount();
381 size_t capacity
= media_codec_
->GetOutputBuffersCapacity();
382 bool ret
= count
<= num_output_buffers_
&&
383 capacity
<= output_buffers_capacity_
;
384 LOG_IF(ERROR
, !ret
) << "Need more/bigger buffers; before: "
385 << num_output_buffers_
<< "x" << output_buffers_capacity_
386 << ", now: " << count
<< "x" << capacity
;
387 UMA_HISTOGRAM_BOOLEAN("Media.AVEA.OutputBuffersSuffice", ret
);
391 void AndroidVideoEncodeAccelerator::DequeueOutput() {
392 if (!client_ptr_factory_
->GetWeakPtr() ||
393 available_bitstream_buffers_
.empty() || num_buffers_at_codec_
== 0) {
400 bool key_frame
= false;
402 media::MediaCodecStatus status
= media_codec_
->DequeueOutputBuffer(
403 NoWaitTimeOut(), &buf_index
, &offset
, &size
, NULL
, NULL
, &key_frame
);
405 case media::MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER
:
408 case media::MEDIA_CODEC_ERROR
:
409 RETURN_ON_FAILURE(false, "Codec error", kPlatformFailureError
);
410 // Unreachable because of previous statement, but included for clarity.
413 case media::MEDIA_CODEC_OUTPUT_FORMAT_CHANGED
: // Fall-through.
414 case media::MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED
:
415 RETURN_ON_FAILURE(DoOutputBuffersSuffice(),
416 "Bitstream now requires more/larger buffers",
417 kPlatformFailureError
);
420 case media::MEDIA_CODEC_OK
:
421 DCHECK_GE(buf_index
, 0);
428 } while (buf_index
< 0);
430 media::BitstreamBuffer bitstream_buffer
= available_bitstream_buffers_
.back();
431 available_bitstream_buffers_
.pop_back();
432 scoped_ptr
<base::SharedMemory
> shm(
433 new base::SharedMemory(bitstream_buffer
.handle(), false));
434 RETURN_ON_FAILURE(shm
->Map(bitstream_buffer
.size()),
436 kPlatformFailureError
);
437 RETURN_ON_FAILURE(size
<= shm
->mapped_size(),
438 "Encoded buffer too large: " << size
<< ">"
439 << shm
->mapped_size(),
440 kPlatformFailureError
);
442 media_codec_
->CopyFromOutputBuffer(buf_index
, offset
, shm
->memory(), size
);
443 media_codec_
->ReleaseOutputBuffer(buf_index
, false);
444 --num_buffers_at_codec_
;
446 UMA_HISTOGRAM_COUNTS_10000("Media.AVEA.EncodedBufferSizeKB", size
/ 1024);
447 base::MessageLoop::current()->PostTask(
449 base::Bind(&VideoEncodeAccelerator::Client::BitstreamBufferReady
,
450 client_ptr_factory_
->GetWeakPtr(),
451 bitstream_buffer
.id(),
456 } // namespace content