Revert "Omit calls to set composing region when pasting image."
[chromium-blink-merge.git] / media / cast / test / fake_media_source.cc
blob7288728ae311dd5c652a084988f25e175f1eaa30
1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/cast/test/fake_media_source.h"
7 #include "base/bind.h"
8 #include "base/files/scoped_file.h"
9 #include "base/logging.h"
10 #include "base/rand_util.h"
11 #include "base/strings/string_number_conversions.h"
12 #include "media/base/audio_buffer.h"
13 #include "media/base/audio_bus.h"
14 #include "media/base/audio_fifo.h"
15 #include "media/base/audio_timestamp_helper.h"
16 #include "media/base/media.h"
17 #include "media/base/video_frame.h"
18 #include "media/base/video_util.h"
19 #include "media/cast/cast_sender.h"
20 #include "media/cast/test/utility/audio_utility.h"
21 #include "media/cast/test/utility/video_utility.h"
22 #include "media/ffmpeg/ffmpeg_common.h"
23 #include "media/ffmpeg/ffmpeg_deleters.h"
24 #include "media/filters/ffmpeg_glue.h"
25 #include "media/filters/in_memory_url_protocol.h"
26 #include "ui/gfx/geometry/size.h"
28 namespace {
30 static const int kSoundFrequency = 440; // Frequency of sinusoid wave.
31 static const float kSoundVolume = 0.10f;
32 static const int kAudioFrameMs = 10; // Each audio frame is exactly 10ms.
33 static const int kAudioPacketsPerSecond = 1000 / kAudioFrameMs;
35 // Bounds for variable frame size mode.
36 static const int kMinFakeFrameWidth = 60;
37 static const int kMinFakeFrameHeight = 34;
38 static const int kStartingFakeFrameWidth = 854;
39 static const int kStartingFakeFrameHeight = 480;
40 static const int kMaxFakeFrameWidth = 1280;
41 static const int kMaxFakeFrameHeight = 720;
42 static const int kMaxFrameSizeChangeMillis = 5000;
44 void AVFreeFrame(AVFrame* frame) {
45 av_frame_free(&frame);
48 base::TimeDelta PtsToTimeDelta(int64 pts, const AVRational& time_base) {
49 return pts * base::TimeDelta::FromSeconds(1) * time_base.num / time_base.den;
52 int64 TimeDeltaToPts(base::TimeDelta delta, const AVRational& time_base) {
53 return static_cast<int64>(
54 delta.InSecondsF() * time_base.den / time_base.num +
55 0.5 /* rounding */);
58 } // namespace
60 namespace media {
61 namespace cast {
63 FakeMediaSource::FakeMediaSource(
64 scoped_refptr<base::SingleThreadTaskRunner> task_runner,
65 base::TickClock* clock,
66 const AudioSenderConfig& audio_config,
67 const VideoSenderConfig& video_config,
68 bool keep_frames)
69 : task_runner_(task_runner),
70 output_audio_params_(AudioParameters::AUDIO_PCM_LINEAR,
71 media::GuessChannelLayout(audio_config.channels),
72 audio_config.frequency,
73 32,
74 audio_config.frequency / kAudioPacketsPerSecond),
75 video_config_(video_config),
76 keep_frames_(keep_frames),
77 variable_frame_size_mode_(false),
78 synthetic_count_(0),
79 clock_(clock),
80 audio_frame_count_(0),
81 video_frame_count_(0),
82 av_format_context_(NULL),
83 audio_stream_index_(-1),
84 playback_rate_(1.0),
85 video_stream_index_(-1),
86 video_frame_rate_numerator_(video_config.max_frame_rate),
87 video_frame_rate_denominator_(1),
88 video_first_pts_(0),
89 video_first_pts_set_(false),
90 weak_factory_(this) {
91 CHECK(output_audio_params_.IsValid());
92 audio_bus_factory_.reset(new TestAudioBusFactory(audio_config.channels,
93 audio_config.frequency,
94 kSoundFrequency,
95 kSoundVolume));
98 FakeMediaSource::~FakeMediaSource() {
101 void FakeMediaSource::SetSourceFile(const base::FilePath& video_file,
102 int final_fps) {
103 DCHECK(!video_file.empty());
105 LOG(INFO) << "Source: " << video_file.value();
106 if (!file_data_.Initialize(video_file)) {
107 LOG(ERROR) << "Cannot load file.";
108 return;
110 protocol_.reset(
111 new InMemoryUrlProtocol(file_data_.data(), file_data_.length(), false));
112 glue_.reset(new FFmpegGlue(protocol_.get()));
114 if (!glue_->OpenContext()) {
115 LOG(ERROR) << "Cannot open file.";
116 return;
119 // AVFormatContext is owned by the glue.
120 av_format_context_ = glue_->format_context();
121 if (avformat_find_stream_info(av_format_context_, NULL) < 0) {
122 LOG(ERROR) << "Cannot find stream information.";
123 return;
126 // Prepare FFmpeg decoders.
127 for (unsigned int i = 0; i < av_format_context_->nb_streams; ++i) {
128 AVStream* av_stream = av_format_context_->streams[i];
129 AVCodecContext* av_codec_context = av_stream->codec;
130 AVCodec* av_codec = avcodec_find_decoder(av_codec_context->codec_id);
132 if (!av_codec) {
133 LOG(ERROR) << "Cannot find decoder for the codec: "
134 << av_codec_context->codec_id;
135 continue;
138 // Number of threads for decoding.
139 av_codec_context->thread_count = 2;
140 av_codec_context->error_concealment = FF_EC_GUESS_MVS | FF_EC_DEBLOCK;
141 av_codec_context->request_sample_fmt = AV_SAMPLE_FMT_S16;
143 if (avcodec_open2(av_codec_context, av_codec, NULL) < 0) {
144 LOG(ERROR) << "Cannot open AVCodecContext for the codec: "
145 << av_codec_context->codec_id;
146 return;
149 if (av_codec->type == AVMEDIA_TYPE_AUDIO) {
150 if (av_codec_context->sample_fmt == AV_SAMPLE_FMT_S16P) {
151 LOG(ERROR) << "Audio format not supported.";
152 continue;
154 ChannelLayout layout = ChannelLayoutToChromeChannelLayout(
155 av_codec_context->channel_layout,
156 av_codec_context->channels);
157 if (layout == CHANNEL_LAYOUT_UNSUPPORTED) {
158 LOG(ERROR) << "Unsupported audio channels layout.";
159 continue;
161 if (audio_stream_index_ != -1) {
162 LOG(WARNING) << "Found multiple audio streams.";
164 audio_stream_index_ = static_cast<int>(i);
165 source_audio_params_.Reset(
166 AudioParameters::AUDIO_PCM_LINEAR,
167 layout,
168 av_codec_context->sample_rate,
169 8 * av_get_bytes_per_sample(av_codec_context->sample_fmt),
170 av_codec_context->sample_rate / kAudioPacketsPerSecond);
171 source_audio_params_.set_channels_for_discrete(
172 av_codec_context->channels);
173 CHECK(source_audio_params_.IsValid());
174 LOG(INFO) << "Source file has audio.";
175 } else if (av_codec->type == AVMEDIA_TYPE_VIDEO) {
176 VideoPixelFormat format =
177 AVPixelFormatToVideoPixelFormat(av_codec_context->pix_fmt);
178 if (format != PIXEL_FORMAT_YV12) {
179 LOG(ERROR) << "Cannot handle non YV12 video format: " << format;
180 continue;
182 if (video_stream_index_ != -1) {
183 LOG(WARNING) << "Found multiple video streams.";
185 video_stream_index_ = static_cast<int>(i);
186 if (final_fps > 0) {
187 // If video is played at a manual speed audio needs to match.
188 playback_rate_ = 1.0 * final_fps *
189 av_stream->r_frame_rate.den / av_stream->r_frame_rate.num;
190 video_frame_rate_numerator_ = final_fps;
191 video_frame_rate_denominator_ = 1;
192 } else {
193 playback_rate_ = 1.0;
194 video_frame_rate_numerator_ = av_stream->r_frame_rate.num;
195 video_frame_rate_denominator_ = av_stream->r_frame_rate.den;
197 LOG(INFO) << "Source file has video.";
198 } else {
199 LOG(ERROR) << "Unknown stream type; ignore.";
203 Rewind();
206 void FakeMediaSource::SetVariableFrameSizeMode(bool enabled) {
207 variable_frame_size_mode_ = enabled;
210 void FakeMediaSource::Start(scoped_refptr<AudioFrameInput> audio_frame_input,
211 scoped_refptr<VideoFrameInput> video_frame_input) {
212 audio_frame_input_ = audio_frame_input;
213 video_frame_input_ = video_frame_input;
215 LOG(INFO) << "Max Frame rate: " << video_config_.max_frame_rate;
216 LOG(INFO) << "Source Frame rate: "
217 << video_frame_rate_numerator_ << "/"
218 << video_frame_rate_denominator_ << " fps.";
219 LOG(INFO) << "Audio playback rate: " << playback_rate_;
221 if (start_time_.is_null())
222 start_time_ = clock_->NowTicks();
224 if (!is_transcoding_audio() && !is_transcoding_video()) {
225 // Send fake patterns.
226 task_runner_->PostTask(
227 FROM_HERE,
228 base::Bind(&FakeMediaSource::SendNextFakeFrame,
229 weak_factory_.GetWeakPtr()));
230 return;
233 // Send transcoding streams.
234 audio_algo_.Initialize(source_audio_params_);
235 audio_algo_.FlushBuffers();
236 audio_fifo_input_bus_ = AudioBus::Create(
237 source_audio_params_.channels(),
238 source_audio_params_.frames_per_buffer());
239 // Audio FIFO can carry all data fron AudioRendererAlgorithm.
240 audio_fifo_.reset(
241 new AudioFifo(source_audio_params_.channels(),
242 audio_algo_.QueueCapacity()));
243 audio_converter_.reset(new media::AudioConverter(
244 source_audio_params_, output_audio_params_, true));
245 audio_converter_->AddInput(this);
246 task_runner_->PostTask(
247 FROM_HERE,
248 base::Bind(&FakeMediaSource::SendNextFrame, weak_factory_.GetWeakPtr()));
251 void FakeMediaSource::SendNextFakeFrame() {
252 UpdateNextFrameSize();
253 scoped_refptr<VideoFrame> video_frame =
254 VideoFrame::CreateBlackFrame(current_frame_size_);
255 PopulateVideoFrame(video_frame.get(), synthetic_count_);
256 ++synthetic_count_;
258 const base::TimeTicks now = clock_->NowTicks();
260 base::TimeDelta video_time = VideoFrameTime(++video_frame_count_);
261 video_frame->set_timestamp(video_time);
262 if (keep_frames_)
263 inserted_video_frame_queue_.push(video_frame);
264 video_frame_input_->InsertRawVideoFrame(video_frame,
265 start_time_ + video_time);
267 // Send just enough audio data to match next video frame's time.
268 base::TimeDelta audio_time = AudioFrameTime(audio_frame_count_);
269 while (audio_time < video_time) {
270 if (is_transcoding_audio()) {
271 Decode(true);
272 CHECK(!audio_bus_queue_.empty()) << "No audio decoded.";
273 scoped_ptr<AudioBus> bus(audio_bus_queue_.front());
274 audio_bus_queue_.pop();
275 audio_frame_input_->InsertAudio(
276 bus.Pass(), start_time_ + audio_time);
277 } else {
278 audio_frame_input_->InsertAudio(
279 audio_bus_factory_->NextAudioBus(
280 base::TimeDelta::FromMilliseconds(kAudioFrameMs)),
281 start_time_ + audio_time);
283 audio_time = AudioFrameTime(++audio_frame_count_);
286 // This is the time since FakeMediaSource was started.
287 const base::TimeDelta elapsed_time = now - start_time_;
289 // Handle the case when frame generation cannot keep up.
290 // Move the time ahead to match the next frame.
291 while (video_time < elapsed_time) {
292 LOG(WARNING) << "Skipping one frame.";
293 video_time = VideoFrameTime(++video_frame_count_);
296 task_runner_->PostDelayedTask(
297 FROM_HERE,
298 base::Bind(&FakeMediaSource::SendNextFakeFrame,
299 weak_factory_.GetWeakPtr()),
300 video_time - elapsed_time);
303 void FakeMediaSource::UpdateNextFrameSize() {
304 if (variable_frame_size_mode_) {
305 bool update_size_change_time = false;
306 if (current_frame_size_.IsEmpty()) {
307 current_frame_size_ = gfx::Size(kStartingFakeFrameWidth,
308 kStartingFakeFrameHeight);
309 update_size_change_time = true;
310 } else if (clock_->NowTicks() >= next_frame_size_change_time_) {
311 current_frame_size_ = gfx::Size(
312 base::RandInt(kMinFakeFrameWidth, kMaxFakeFrameWidth),
313 base::RandInt(kMinFakeFrameHeight, kMaxFakeFrameHeight));
314 update_size_change_time = true;
317 if (update_size_change_time) {
318 next_frame_size_change_time_ = clock_->NowTicks() +
319 base::TimeDelta::FromMillisecondsD(
320 base::RandDouble() * kMaxFrameSizeChangeMillis);
322 } else {
323 current_frame_size_ = gfx::Size(kStartingFakeFrameWidth,
324 kStartingFakeFrameHeight);
325 next_frame_size_change_time_ = base::TimeTicks();
329 bool FakeMediaSource::SendNextTranscodedVideo(base::TimeDelta elapsed_time) {
330 if (!is_transcoding_video())
331 return false;
333 Decode(false);
334 if (video_frame_queue_.empty())
335 return false;
337 const scoped_refptr<VideoFrame> video_frame = video_frame_queue_.front();
338 if (elapsed_time < video_frame->timestamp())
339 return false;
340 video_frame_queue_.pop();
342 // Use the timestamp from the file if we're transcoding.
343 video_frame->set_timestamp(ScaleTimestamp(video_frame->timestamp()));
344 if (keep_frames_)
345 inserted_video_frame_queue_.push(video_frame);
346 video_frame_input_->InsertRawVideoFrame(
347 video_frame, start_time_ + video_frame->timestamp());
349 // Make sure queue is not empty.
350 Decode(false);
351 return true;
354 bool FakeMediaSource::SendNextTranscodedAudio(base::TimeDelta elapsed_time) {
355 if (!is_transcoding_audio())
356 return false;
358 Decode(true);
359 if (audio_bus_queue_.empty())
360 return false;
362 base::TimeDelta audio_time = audio_sent_ts_->GetTimestamp();
363 if (elapsed_time < audio_time)
364 return false;
365 scoped_ptr<AudioBus> bus(audio_bus_queue_.front());
366 audio_bus_queue_.pop();
367 audio_sent_ts_->AddFrames(bus->frames());
368 audio_frame_input_->InsertAudio(
369 bus.Pass(), start_time_ + audio_time);
371 // Make sure queue is not empty.
372 Decode(true);
373 return true;
376 void FakeMediaSource::SendNextFrame() {
377 // Send as much as possible. Audio is sent according to
378 // system time.
379 while (SendNextTranscodedAudio(clock_->NowTicks() - start_time_)) {
382 // Video is sync'ed to audio.
383 while (SendNextTranscodedVideo(audio_sent_ts_->GetTimestamp())) {
386 if (audio_bus_queue_.empty() && video_frame_queue_.empty()) {
387 // Both queues are empty can only mean that we have reached
388 // the end of the stream.
389 LOG(INFO) << "Rewind.";
390 Rewind();
393 // Send next send.
394 task_runner_->PostDelayedTask(
395 FROM_HERE,
396 base::Bind(&FakeMediaSource::SendNextFrame, weak_factory_.GetWeakPtr()),
397 base::TimeDelta::FromMilliseconds(kAudioFrameMs));
400 base::TimeDelta FakeMediaSource::VideoFrameTime(int frame_number) {
401 return frame_number * base::TimeDelta::FromSeconds(1) *
402 video_frame_rate_denominator_ / video_frame_rate_numerator_;
405 base::TimeDelta FakeMediaSource::ScaleTimestamp(base::TimeDelta timestamp) {
406 return base::TimeDelta::FromSecondsD(timestamp.InSecondsF() / playback_rate_);
409 base::TimeDelta FakeMediaSource::AudioFrameTime(int frame_number) {
410 return frame_number * base::TimeDelta::FromMilliseconds(kAudioFrameMs);
413 void FakeMediaSource::Rewind() {
414 CHECK(av_seek_frame(av_format_context_, -1, 0, AVSEEK_FLAG_BACKWARD) >= 0)
415 << "Failed to rewind to the beginning.";
418 ScopedAVPacket FakeMediaSource::DemuxOnePacket(bool* audio) {
419 ScopedAVPacket packet(new AVPacket());
420 if (av_read_frame(av_format_context_, packet.get()) < 0) {
421 VLOG(1) << "Failed to read one AVPacket.";
422 packet.reset();
423 return packet.Pass();
426 int stream_index = static_cast<int>(packet->stream_index);
427 if (stream_index == audio_stream_index_) {
428 *audio = true;
429 } else if (stream_index == video_stream_index_) {
430 *audio = false;
431 } else {
432 // Ignore unknown packet.
433 LOG(INFO) << "Unknown packet.";
434 packet.reset();
436 return packet.Pass();
439 void FakeMediaSource::DecodeAudio(ScopedAVPacket packet) {
440 // Audio.
441 AVFrame* avframe = av_frame_alloc();
443 // Make a shallow copy of packet so we can slide packet.data as frames are
444 // decoded from the packet; otherwise av_free_packet() will corrupt memory.
445 AVPacket packet_temp = *packet.get();
447 do {
448 int frame_decoded = 0;
449 int result = avcodec_decode_audio4(
450 av_audio_context(), avframe, &frame_decoded, &packet_temp);
451 CHECK(result >= 0) << "Failed to decode audio.";
452 packet_temp.size -= result;
453 packet_temp.data += result;
454 if (!frame_decoded)
455 continue;
457 int frames_read = avframe->nb_samples;
458 if (frames_read < 0)
459 break;
461 if (!audio_sent_ts_) {
462 // Initialize the base time to the first packet in the file.
463 // This is set to the frequency we send to the receiver.
464 // Not the frequency of the source file. This is because we
465 // increment the frame count by samples we sent.
466 audio_sent_ts_.reset(
467 new AudioTimestampHelper(output_audio_params_.sample_rate()));
468 // For some files this is an invalid value.
469 base::TimeDelta base_ts;
470 audio_sent_ts_->SetBaseTimestamp(base_ts);
473 scoped_refptr<AudioBuffer> buffer =
474 AudioBuffer::CopyFrom(
475 AVSampleFormatToSampleFormat(
476 av_audio_context()->sample_fmt),
477 ChannelLayoutToChromeChannelLayout(
478 av_audio_context()->channel_layout,
479 av_audio_context()->channels),
480 av_audio_context()->channels,
481 av_audio_context()->sample_rate,
482 frames_read,
483 &avframe->data[0],
484 PtsToTimeDelta(avframe->pkt_pts, av_audio_stream()->time_base));
485 audio_algo_.EnqueueBuffer(buffer);
486 av_frame_unref(avframe);
487 } while (packet_temp.size > 0);
488 av_frame_free(&avframe);
490 const int frames_needed_to_scale =
491 playback_rate_ * av_audio_context()->sample_rate /
492 kAudioPacketsPerSecond;
493 while (frames_needed_to_scale <= audio_algo_.frames_buffered()) {
494 if (!audio_algo_.FillBuffer(audio_fifo_input_bus_.get(), 0,
495 audio_fifo_input_bus_->frames(),
496 playback_rate_)) {
497 // Nothing can be scaled. Decode some more.
498 return;
501 // Prevent overflow of audio data in the FIFO.
502 if (audio_fifo_input_bus_->frames() + audio_fifo_->frames()
503 <= audio_fifo_->max_frames()) {
504 audio_fifo_->Push(audio_fifo_input_bus_.get());
505 } else {
506 LOG(WARNING) << "Audio FIFO full; dropping samples.";
509 // Make sure there's enough data to resample audio.
510 if (audio_fifo_->frames() <
511 2 * source_audio_params_.sample_rate() / kAudioPacketsPerSecond) {
512 continue;
515 scoped_ptr<media::AudioBus> resampled_bus(
516 media::AudioBus::Create(
517 output_audio_params_.channels(),
518 output_audio_params_.sample_rate() / kAudioPacketsPerSecond));
519 audio_converter_->Convert(resampled_bus.get());
520 audio_bus_queue_.push(resampled_bus.release());
524 void FakeMediaSource::DecodeVideo(ScopedAVPacket packet) {
525 // Video.
526 int got_picture;
527 AVFrame* avframe = av_frame_alloc();
528 CHECK(avcodec_decode_video2(
529 av_video_context(), avframe, &got_picture, packet.get()) >= 0)
530 << "Video decode error.";
531 if (!got_picture) {
532 av_frame_free(&avframe);
533 return;
535 gfx::Size size(av_video_context()->width, av_video_context()->height);
537 if (!video_first_pts_set_) {
538 video_first_pts_ = avframe->pkt_pts;
539 video_first_pts_set_ = true;
541 const AVRational& time_base = av_video_stream()->time_base;
542 base::TimeDelta timestamp =
543 PtsToTimeDelta(avframe->pkt_pts - video_first_pts_, time_base);
544 if (timestamp < last_video_frame_timestamp_) {
545 // Stream has rewound. Rebase |video_first_pts_|.
546 const AVRational& frame_rate = av_video_stream()->r_frame_rate;
547 timestamp = last_video_frame_timestamp_ +
548 (base::TimeDelta::FromSeconds(1) * frame_rate.den / frame_rate.num);
549 const int64 adjustment_pts = TimeDeltaToPts(timestamp, time_base);
550 video_first_pts_ = avframe->pkt_pts - adjustment_pts;
553 video_frame_queue_.push(VideoFrame::WrapExternalYuvData(
554 media::PIXEL_FORMAT_YV12, size, gfx::Rect(size), size,
555 avframe->linesize[0], avframe->linesize[1], avframe->linesize[2],
556 avframe->data[0], avframe->data[1], avframe->data[2], timestamp));
557 video_frame_queue_.back()->AddDestructionObserver(
558 base::Bind(&AVFreeFrame, avframe));
559 last_video_frame_timestamp_ = timestamp;
562 void FakeMediaSource::Decode(bool decode_audio) {
563 // Read the stream until one video frame can be decoded.
564 while (true) {
565 if (decode_audio && !audio_bus_queue_.empty())
566 return;
567 if (!decode_audio && !video_frame_queue_.empty())
568 return;
570 bool audio_packet = false;
571 ScopedAVPacket packet = DemuxOnePacket(&audio_packet);
572 if (!packet) {
573 VLOG(1) << "End of stream.";
574 return;
577 if (audio_packet)
578 DecodeAudio(packet.Pass());
579 else
580 DecodeVideo(packet.Pass());
584 double FakeMediaSource::ProvideInput(media::AudioBus* output_bus,
585 base::TimeDelta buffer_delay) {
586 if (audio_fifo_->frames() >= output_bus->frames()) {
587 audio_fifo_->Consume(output_bus, 0, output_bus->frames());
588 return 1.0;
589 } else {
590 LOG(WARNING) << "Not enough audio data for resampling.";
591 output_bus->Zero();
592 return 0.0;
596 scoped_refptr<media::VideoFrame>
597 FakeMediaSource::PopOldestInsertedVideoFrame() {
598 CHECK(!inserted_video_frame_queue_.empty());
599 scoped_refptr<media::VideoFrame> video_frame =
600 inserted_video_frame_queue_.front();
601 inserted_video_frame_queue_.pop();
602 return video_frame;
605 AVStream* FakeMediaSource::av_audio_stream() {
606 return av_format_context_->streams[audio_stream_index_];
609 AVStream* FakeMediaSource::av_video_stream() {
610 return av_format_context_->streams[video_stream_index_];
613 AVCodecContext* FakeMediaSource::av_audio_context() {
614 return av_audio_stream()->codec;
617 AVCodecContext* FakeMediaSource::av_video_context() {
618 return av_video_stream()->codec;
621 } // namespace cast
622 } // namespace media