1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/cast/test/fake_media_source.h"
8 #include "base/files/scoped_file.h"
9 #include "base/logging.h"
10 #include "base/rand_util.h"
11 #include "base/strings/string_number_conversions.h"
12 #include "media/base/audio_buffer.h"
13 #include "media/base/audio_bus.h"
14 #include "media/base/audio_fifo.h"
15 #include "media/base/audio_timestamp_helper.h"
16 #include "media/base/media.h"
17 #include "media/base/video_frame.h"
18 #include "media/base/video_util.h"
19 #include "media/cast/cast_sender.h"
20 #include "media/cast/test/utility/audio_utility.h"
21 #include "media/cast/test/utility/video_utility.h"
22 #include "media/ffmpeg/ffmpeg_common.h"
23 #include "media/ffmpeg/ffmpeg_deleters.h"
24 #include "media/filters/ffmpeg_glue.h"
25 #include "media/filters/in_memory_url_protocol.h"
26 #include "ui/gfx/geometry/size.h"
30 static const int kSoundFrequency
= 440; // Frequency of sinusoid wave.
31 static const float kSoundVolume
= 0.10f
;
32 static const int kAudioFrameMs
= 10; // Each audio frame is exactly 10ms.
33 static const int kAudioPacketsPerSecond
= 1000 / kAudioFrameMs
;
35 // Bounds for variable frame size mode.
36 static const int kMinFakeFrameWidth
= 60;
37 static const int kMinFakeFrameHeight
= 34;
38 static const int kStartingFakeFrameWidth
= 854;
39 static const int kStartingFakeFrameHeight
= 480;
40 static const int kMaxFakeFrameWidth
= 1280;
41 static const int kMaxFakeFrameHeight
= 720;
42 static const int kMaxFrameSizeChangeMillis
= 5000;
44 void AVFreeFrame(AVFrame
* frame
) {
45 av_frame_free(&frame
);
48 base::TimeDelta
PtsToTimeDelta(int64 pts
, const AVRational
& time_base
) {
49 return pts
* base::TimeDelta::FromSeconds(1) * time_base
.num
/ time_base
.den
;
52 int64
TimeDeltaToPts(base::TimeDelta delta
, const AVRational
& time_base
) {
53 return static_cast<int64
>(
54 delta
.InSecondsF() * time_base
.den
/ time_base
.num
+
63 FakeMediaSource::FakeMediaSource(
64 scoped_refptr
<base::SingleThreadTaskRunner
> task_runner
,
65 base::TickClock
* clock
,
66 const AudioSenderConfig
& audio_config
,
67 const VideoSenderConfig
& video_config
,
69 : task_runner_(task_runner
),
70 output_audio_params_(AudioParameters::AUDIO_PCM_LINEAR
,
71 media::GuessChannelLayout(audio_config
.channels
),
72 audio_config
.frequency
,
74 audio_config
.frequency
/ kAudioPacketsPerSecond
),
75 video_config_(video_config
),
76 keep_frames_(keep_frames
),
77 variable_frame_size_mode_(false),
80 audio_frame_count_(0),
81 video_frame_count_(0),
82 av_format_context_(NULL
),
83 audio_stream_index_(-1),
85 video_stream_index_(-1),
86 video_frame_rate_numerator_(video_config
.max_frame_rate
),
87 video_frame_rate_denominator_(1),
89 video_first_pts_set_(false),
91 CHECK(output_audio_params_
.IsValid());
92 audio_bus_factory_
.reset(new TestAudioBusFactory(audio_config
.channels
,
93 audio_config
.frequency
,
98 FakeMediaSource::~FakeMediaSource() {
101 void FakeMediaSource::SetSourceFile(const base::FilePath
& video_file
,
103 DCHECK(!video_file
.empty());
105 LOG(INFO
) << "Source: " << video_file
.value();
106 if (!file_data_
.Initialize(video_file
)) {
107 LOG(ERROR
) << "Cannot load file.";
111 new InMemoryUrlProtocol(file_data_
.data(), file_data_
.length(), false));
112 glue_
.reset(new FFmpegGlue(protocol_
.get()));
114 if (!glue_
->OpenContext()) {
115 LOG(ERROR
) << "Cannot open file.";
119 // AVFormatContext is owned by the glue.
120 av_format_context_
= glue_
->format_context();
121 if (avformat_find_stream_info(av_format_context_
, NULL
) < 0) {
122 LOG(ERROR
) << "Cannot find stream information.";
126 // Prepare FFmpeg decoders.
127 for (unsigned int i
= 0; i
< av_format_context_
->nb_streams
; ++i
) {
128 AVStream
* av_stream
= av_format_context_
->streams
[i
];
129 AVCodecContext
* av_codec_context
= av_stream
->codec
;
130 AVCodec
* av_codec
= avcodec_find_decoder(av_codec_context
->codec_id
);
133 LOG(ERROR
) << "Cannot find decoder for the codec: "
134 << av_codec_context
->codec_id
;
138 // Number of threads for decoding.
139 av_codec_context
->thread_count
= 2;
140 av_codec_context
->error_concealment
= FF_EC_GUESS_MVS
| FF_EC_DEBLOCK
;
141 av_codec_context
->request_sample_fmt
= AV_SAMPLE_FMT_S16
;
143 if (avcodec_open2(av_codec_context
, av_codec
, NULL
) < 0) {
144 LOG(ERROR
) << "Cannot open AVCodecContext for the codec: "
145 << av_codec_context
->codec_id
;
149 if (av_codec
->type
== AVMEDIA_TYPE_AUDIO
) {
150 if (av_codec_context
->sample_fmt
== AV_SAMPLE_FMT_S16P
) {
151 LOG(ERROR
) << "Audio format not supported.";
154 ChannelLayout layout
= ChannelLayoutToChromeChannelLayout(
155 av_codec_context
->channel_layout
,
156 av_codec_context
->channels
);
157 if (layout
== CHANNEL_LAYOUT_UNSUPPORTED
) {
158 LOG(ERROR
) << "Unsupported audio channels layout.";
161 if (audio_stream_index_
!= -1) {
162 LOG(WARNING
) << "Found multiple audio streams.";
164 audio_stream_index_
= static_cast<int>(i
);
165 source_audio_params_
.Reset(
166 AudioParameters::AUDIO_PCM_LINEAR
,
168 av_codec_context
->sample_rate
,
169 8 * av_get_bytes_per_sample(av_codec_context
->sample_fmt
),
170 av_codec_context
->sample_rate
/ kAudioPacketsPerSecond
);
171 source_audio_params_
.set_channels_for_discrete(
172 av_codec_context
->channels
);
173 CHECK(source_audio_params_
.IsValid());
174 LOG(INFO
) << "Source file has audio.";
175 } else if (av_codec
->type
== AVMEDIA_TYPE_VIDEO
) {
176 VideoPixelFormat format
=
177 AVPixelFormatToVideoPixelFormat(av_codec_context
->pix_fmt
);
178 if (format
!= PIXEL_FORMAT_YV12
) {
179 LOG(ERROR
) << "Cannot handle non YV12 video format: " << format
;
182 if (video_stream_index_
!= -1) {
183 LOG(WARNING
) << "Found multiple video streams.";
185 video_stream_index_
= static_cast<int>(i
);
187 // If video is played at a manual speed audio needs to match.
188 playback_rate_
= 1.0 * final_fps
*
189 av_stream
->r_frame_rate
.den
/ av_stream
->r_frame_rate
.num
;
190 video_frame_rate_numerator_
= final_fps
;
191 video_frame_rate_denominator_
= 1;
193 playback_rate_
= 1.0;
194 video_frame_rate_numerator_
= av_stream
->r_frame_rate
.num
;
195 video_frame_rate_denominator_
= av_stream
->r_frame_rate
.den
;
197 LOG(INFO
) << "Source file has video.";
199 LOG(ERROR
) << "Unknown stream type; ignore.";
206 void FakeMediaSource::SetVariableFrameSizeMode(bool enabled
) {
207 variable_frame_size_mode_
= enabled
;
210 void FakeMediaSource::Start(scoped_refptr
<AudioFrameInput
> audio_frame_input
,
211 scoped_refptr
<VideoFrameInput
> video_frame_input
) {
212 audio_frame_input_
= audio_frame_input
;
213 video_frame_input_
= video_frame_input
;
215 LOG(INFO
) << "Max Frame rate: " << video_config_
.max_frame_rate
;
216 LOG(INFO
) << "Source Frame rate: "
217 << video_frame_rate_numerator_
<< "/"
218 << video_frame_rate_denominator_
<< " fps.";
219 LOG(INFO
) << "Audio playback rate: " << playback_rate_
;
221 if (start_time_
.is_null())
222 start_time_
= clock_
->NowTicks();
224 if (!is_transcoding_audio() && !is_transcoding_video()) {
225 // Send fake patterns.
226 task_runner_
->PostTask(
228 base::Bind(&FakeMediaSource::SendNextFakeFrame
,
229 weak_factory_
.GetWeakPtr()));
233 // Send transcoding streams.
234 audio_algo_
.Initialize(source_audio_params_
);
235 audio_algo_
.FlushBuffers();
236 audio_fifo_input_bus_
= AudioBus::Create(
237 source_audio_params_
.channels(),
238 source_audio_params_
.frames_per_buffer());
239 // Audio FIFO can carry all data fron AudioRendererAlgorithm.
241 new AudioFifo(source_audio_params_
.channels(),
242 audio_algo_
.QueueCapacity()));
243 audio_converter_
.reset(new media::AudioConverter(
244 source_audio_params_
, output_audio_params_
, true));
245 audio_converter_
->AddInput(this);
246 task_runner_
->PostTask(
248 base::Bind(&FakeMediaSource::SendNextFrame
, weak_factory_
.GetWeakPtr()));
251 void FakeMediaSource::SendNextFakeFrame() {
252 UpdateNextFrameSize();
253 scoped_refptr
<VideoFrame
> video_frame
=
254 VideoFrame::CreateBlackFrame(current_frame_size_
);
255 PopulateVideoFrame(video_frame
.get(), synthetic_count_
);
258 const base::TimeTicks now
= clock_
->NowTicks();
260 base::TimeDelta video_time
= VideoFrameTime(++video_frame_count_
);
261 video_frame
->set_timestamp(video_time
);
263 inserted_video_frame_queue_
.push(video_frame
);
264 video_frame_input_
->InsertRawVideoFrame(video_frame
,
265 start_time_
+ video_time
);
267 // Send just enough audio data to match next video frame's time.
268 base::TimeDelta audio_time
= AudioFrameTime(audio_frame_count_
);
269 while (audio_time
< video_time
) {
270 if (is_transcoding_audio()) {
272 CHECK(!audio_bus_queue_
.empty()) << "No audio decoded.";
273 scoped_ptr
<AudioBus
> bus(audio_bus_queue_
.front());
274 audio_bus_queue_
.pop();
275 audio_frame_input_
->InsertAudio(
276 bus
.Pass(), start_time_
+ audio_time
);
278 audio_frame_input_
->InsertAudio(
279 audio_bus_factory_
->NextAudioBus(
280 base::TimeDelta::FromMilliseconds(kAudioFrameMs
)),
281 start_time_
+ audio_time
);
283 audio_time
= AudioFrameTime(++audio_frame_count_
);
286 // This is the time since FakeMediaSource was started.
287 const base::TimeDelta elapsed_time
= now
- start_time_
;
289 // Handle the case when frame generation cannot keep up.
290 // Move the time ahead to match the next frame.
291 while (video_time
< elapsed_time
) {
292 LOG(WARNING
) << "Skipping one frame.";
293 video_time
= VideoFrameTime(++video_frame_count_
);
296 task_runner_
->PostDelayedTask(
298 base::Bind(&FakeMediaSource::SendNextFakeFrame
,
299 weak_factory_
.GetWeakPtr()),
300 video_time
- elapsed_time
);
303 void FakeMediaSource::UpdateNextFrameSize() {
304 if (variable_frame_size_mode_
) {
305 bool update_size_change_time
= false;
306 if (current_frame_size_
.IsEmpty()) {
307 current_frame_size_
= gfx::Size(kStartingFakeFrameWidth
,
308 kStartingFakeFrameHeight
);
309 update_size_change_time
= true;
310 } else if (clock_
->NowTicks() >= next_frame_size_change_time_
) {
311 current_frame_size_
= gfx::Size(
312 base::RandInt(kMinFakeFrameWidth
, kMaxFakeFrameWidth
),
313 base::RandInt(kMinFakeFrameHeight
, kMaxFakeFrameHeight
));
314 update_size_change_time
= true;
317 if (update_size_change_time
) {
318 next_frame_size_change_time_
= clock_
->NowTicks() +
319 base::TimeDelta::FromMillisecondsD(
320 base::RandDouble() * kMaxFrameSizeChangeMillis
);
323 current_frame_size_
= gfx::Size(kStartingFakeFrameWidth
,
324 kStartingFakeFrameHeight
);
325 next_frame_size_change_time_
= base::TimeTicks();
329 bool FakeMediaSource::SendNextTranscodedVideo(base::TimeDelta elapsed_time
) {
330 if (!is_transcoding_video())
334 if (video_frame_queue_
.empty())
337 const scoped_refptr
<VideoFrame
> video_frame
= video_frame_queue_
.front();
338 if (elapsed_time
< video_frame
->timestamp())
340 video_frame_queue_
.pop();
342 // Use the timestamp from the file if we're transcoding.
343 video_frame
->set_timestamp(ScaleTimestamp(video_frame
->timestamp()));
345 inserted_video_frame_queue_
.push(video_frame
);
346 video_frame_input_
->InsertRawVideoFrame(
347 video_frame
, start_time_
+ video_frame
->timestamp());
349 // Make sure queue is not empty.
354 bool FakeMediaSource::SendNextTranscodedAudio(base::TimeDelta elapsed_time
) {
355 if (!is_transcoding_audio())
359 if (audio_bus_queue_
.empty())
362 base::TimeDelta audio_time
= audio_sent_ts_
->GetTimestamp();
363 if (elapsed_time
< audio_time
)
365 scoped_ptr
<AudioBus
> bus(audio_bus_queue_
.front());
366 audio_bus_queue_
.pop();
367 audio_sent_ts_
->AddFrames(bus
->frames());
368 audio_frame_input_
->InsertAudio(
369 bus
.Pass(), start_time_
+ audio_time
);
371 // Make sure queue is not empty.
376 void FakeMediaSource::SendNextFrame() {
377 // Send as much as possible. Audio is sent according to
379 while (SendNextTranscodedAudio(clock_
->NowTicks() - start_time_
)) {
382 // Video is sync'ed to audio.
383 while (SendNextTranscodedVideo(audio_sent_ts_
->GetTimestamp())) {
386 if (audio_bus_queue_
.empty() && video_frame_queue_
.empty()) {
387 // Both queues are empty can only mean that we have reached
388 // the end of the stream.
389 LOG(INFO
) << "Rewind.";
394 task_runner_
->PostDelayedTask(
396 base::Bind(&FakeMediaSource::SendNextFrame
, weak_factory_
.GetWeakPtr()),
397 base::TimeDelta::FromMilliseconds(kAudioFrameMs
));
400 base::TimeDelta
FakeMediaSource::VideoFrameTime(int frame_number
) {
401 return frame_number
* base::TimeDelta::FromSeconds(1) *
402 video_frame_rate_denominator_
/ video_frame_rate_numerator_
;
405 base::TimeDelta
FakeMediaSource::ScaleTimestamp(base::TimeDelta timestamp
) {
406 return base::TimeDelta::FromSecondsD(timestamp
.InSecondsF() / playback_rate_
);
409 base::TimeDelta
FakeMediaSource::AudioFrameTime(int frame_number
) {
410 return frame_number
* base::TimeDelta::FromMilliseconds(kAudioFrameMs
);
413 void FakeMediaSource::Rewind() {
414 CHECK(av_seek_frame(av_format_context_
, -1, 0, AVSEEK_FLAG_BACKWARD
) >= 0)
415 << "Failed to rewind to the beginning.";
418 ScopedAVPacket
FakeMediaSource::DemuxOnePacket(bool* audio
) {
419 ScopedAVPacket
packet(new AVPacket());
420 if (av_read_frame(av_format_context_
, packet
.get()) < 0) {
421 VLOG(1) << "Failed to read one AVPacket.";
423 return packet
.Pass();
426 int stream_index
= static_cast<int>(packet
->stream_index
);
427 if (stream_index
== audio_stream_index_
) {
429 } else if (stream_index
== video_stream_index_
) {
432 // Ignore unknown packet.
433 LOG(INFO
) << "Unknown packet.";
436 return packet
.Pass();
439 void FakeMediaSource::DecodeAudio(ScopedAVPacket packet
) {
441 AVFrame
* avframe
= av_frame_alloc();
443 // Make a shallow copy of packet so we can slide packet.data as frames are
444 // decoded from the packet; otherwise av_free_packet() will corrupt memory.
445 AVPacket packet_temp
= *packet
.get();
448 int frame_decoded
= 0;
449 int result
= avcodec_decode_audio4(
450 av_audio_context(), avframe
, &frame_decoded
, &packet_temp
);
451 CHECK(result
>= 0) << "Failed to decode audio.";
452 packet_temp
.size
-= result
;
453 packet_temp
.data
+= result
;
457 int frames_read
= avframe
->nb_samples
;
461 if (!audio_sent_ts_
) {
462 // Initialize the base time to the first packet in the file.
463 // This is set to the frequency we send to the receiver.
464 // Not the frequency of the source file. This is because we
465 // increment the frame count by samples we sent.
466 audio_sent_ts_
.reset(
467 new AudioTimestampHelper(output_audio_params_
.sample_rate()));
468 // For some files this is an invalid value.
469 base::TimeDelta base_ts
;
470 audio_sent_ts_
->SetBaseTimestamp(base_ts
);
473 scoped_refptr
<AudioBuffer
> buffer
=
474 AudioBuffer::CopyFrom(
475 AVSampleFormatToSampleFormat(
476 av_audio_context()->sample_fmt
),
477 ChannelLayoutToChromeChannelLayout(
478 av_audio_context()->channel_layout
,
479 av_audio_context()->channels
),
480 av_audio_context()->channels
,
481 av_audio_context()->sample_rate
,
484 PtsToTimeDelta(avframe
->pkt_pts
, av_audio_stream()->time_base
));
485 audio_algo_
.EnqueueBuffer(buffer
);
486 av_frame_unref(avframe
);
487 } while (packet_temp
.size
> 0);
488 av_frame_free(&avframe
);
490 const int frames_needed_to_scale
=
491 playback_rate_
* av_audio_context()->sample_rate
/
492 kAudioPacketsPerSecond
;
493 while (frames_needed_to_scale
<= audio_algo_
.frames_buffered()) {
494 if (!audio_algo_
.FillBuffer(audio_fifo_input_bus_
.get(), 0,
495 audio_fifo_input_bus_
->frames(),
497 // Nothing can be scaled. Decode some more.
501 // Prevent overflow of audio data in the FIFO.
502 if (audio_fifo_input_bus_
->frames() + audio_fifo_
->frames()
503 <= audio_fifo_
->max_frames()) {
504 audio_fifo_
->Push(audio_fifo_input_bus_
.get());
506 LOG(WARNING
) << "Audio FIFO full; dropping samples.";
509 // Make sure there's enough data to resample audio.
510 if (audio_fifo_
->frames() <
511 2 * source_audio_params_
.sample_rate() / kAudioPacketsPerSecond
) {
515 scoped_ptr
<media::AudioBus
> resampled_bus(
516 media::AudioBus::Create(
517 output_audio_params_
.channels(),
518 output_audio_params_
.sample_rate() / kAudioPacketsPerSecond
));
519 audio_converter_
->Convert(resampled_bus
.get());
520 audio_bus_queue_
.push(resampled_bus
.release());
524 void FakeMediaSource::DecodeVideo(ScopedAVPacket packet
) {
527 AVFrame
* avframe
= av_frame_alloc();
528 CHECK(avcodec_decode_video2(
529 av_video_context(), avframe
, &got_picture
, packet
.get()) >= 0)
530 << "Video decode error.";
532 av_frame_free(&avframe
);
535 gfx::Size
size(av_video_context()->width
, av_video_context()->height
);
537 if (!video_first_pts_set_
) {
538 video_first_pts_
= avframe
->pkt_pts
;
539 video_first_pts_set_
= true;
541 const AVRational
& time_base
= av_video_stream()->time_base
;
542 base::TimeDelta timestamp
=
543 PtsToTimeDelta(avframe
->pkt_pts
- video_first_pts_
, time_base
);
544 if (timestamp
< last_video_frame_timestamp_
) {
545 // Stream has rewound. Rebase |video_first_pts_|.
546 const AVRational
& frame_rate
= av_video_stream()->r_frame_rate
;
547 timestamp
= last_video_frame_timestamp_
+
548 (base::TimeDelta::FromSeconds(1) * frame_rate
.den
/ frame_rate
.num
);
549 const int64 adjustment_pts
= TimeDeltaToPts(timestamp
, time_base
);
550 video_first_pts_
= avframe
->pkt_pts
- adjustment_pts
;
553 video_frame_queue_
.push(VideoFrame::WrapExternalYuvData(
554 media::PIXEL_FORMAT_YV12
, size
, gfx::Rect(size
), size
,
555 avframe
->linesize
[0], avframe
->linesize
[1], avframe
->linesize
[2],
556 avframe
->data
[0], avframe
->data
[1], avframe
->data
[2], timestamp
));
557 video_frame_queue_
.back()->AddDestructionObserver(
558 base::Bind(&AVFreeFrame
, avframe
));
559 last_video_frame_timestamp_
= timestamp
;
562 void FakeMediaSource::Decode(bool decode_audio
) {
563 // Read the stream until one video frame can be decoded.
565 if (decode_audio
&& !audio_bus_queue_
.empty())
567 if (!decode_audio
&& !video_frame_queue_
.empty())
570 bool audio_packet
= false;
571 ScopedAVPacket packet
= DemuxOnePacket(&audio_packet
);
573 VLOG(1) << "End of stream.";
578 DecodeAudio(packet
.Pass());
580 DecodeVideo(packet
.Pass());
584 double FakeMediaSource::ProvideInput(media::AudioBus
* output_bus
,
585 base::TimeDelta buffer_delay
) {
586 if (audio_fifo_
->frames() >= output_bus
->frames()) {
587 audio_fifo_
->Consume(output_bus
, 0, output_bus
->frames());
590 LOG(WARNING
) << "Not enough audio data for resampling.";
596 scoped_refptr
<media::VideoFrame
>
597 FakeMediaSource::PopOldestInsertedVideoFrame() {
598 CHECK(!inserted_video_frame_queue_
.empty());
599 scoped_refptr
<media::VideoFrame
> video_frame
=
600 inserted_video_frame_queue_
.front();
601 inserted_video_frame_queue_
.pop();
605 AVStream
* FakeMediaSource::av_audio_stream() {
606 return av_format_context_
->streams
[audio_stream_index_
];
609 AVStream
* FakeMediaSource::av_video_stream() {
610 return av_format_context_
->streams
[video_stream_index_
];
613 AVCodecContext
* FakeMediaSource::av_audio_context() {
614 return av_audio_stream()->codec
;
617 AVCodecContext
* FakeMediaSource::av_video_context() {
618 return av_video_stream()->codec
;