1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
14 #include "base/at_exit.h"
15 #include "base/command_line.h"
16 #include "base/logging.h"
17 #include "base/memory/ref_counted.h"
18 #include "base/memory/scoped_ptr.h"
19 #include "base/message_loop/message_loop.h"
20 #include "base/synchronization/lock.h"
21 #include "base/synchronization/waitable_event.h"
22 #include "base/threading/thread.h"
23 #include "base/time/default_tick_clock.h"
24 #include "base/timer/timer.h"
25 #include "media/audio/audio_io.h"
26 #include "media/audio/audio_manager.h"
27 #include "media/audio/audio_parameters.h"
28 #include "media/audio/fake_audio_log_factory.h"
29 #include "media/base/audio_bus.h"
30 #include "media/base/channel_layout.h"
31 #include "media/base/video_frame.h"
32 #include "media/cast/cast_config.h"
33 #include "media/cast/cast_environment.h"
34 #include "media/cast/cast_receiver.h"
35 #include "media/cast/logging/logging_defines.h"
36 #include "media/cast/test/utility/audio_utility.h"
37 #include "media/cast/test/utility/barcode.h"
38 #include "media/cast/test/utility/default_config.h"
39 #include "media/cast/test/utility/in_process_receiver.h"
40 #include "media/cast/test/utility/input_builder.h"
41 #include "media/cast/test/utility/standalone_cast_environment.h"
42 #include "media/cast/transport/transport/udp_transport.h"
43 #include "net/base/net_util.h"
46 #include "media/cast/test/linux_output_window.h"
52 // Settings chosen to match default sender settings.
53 #define DEFAULT_SEND_PORT "0"
54 #define DEFAULT_RECEIVE_PORT "2344"
55 #define DEFAULT_SEND_IP "0.0.0.0"
56 #define DEFAULT_AUDIO_FEEDBACK_SSRC "2"
57 #define DEFAULT_AUDIO_INCOMING_SSRC "1"
58 #define DEFAULT_AUDIO_PAYLOAD_TYPE "127"
59 #define DEFAULT_VIDEO_FEEDBACK_SSRC "12"
60 #define DEFAULT_VIDEO_INCOMING_SSRC "11"
61 #define DEFAULT_VIDEO_PAYLOAD_TYPE "96"
64 const char* kVideoWindowWidth
= "1280";
65 const char* kVideoWindowHeight
= "720";
68 void GetPorts(int* tx_port
, int* rx_port
) {
69 test::InputBuilder
tx_input(
70 "Enter send port.", DEFAULT_SEND_PORT
, 1, INT_MAX
);
71 *tx_port
= tx_input
.GetIntInput();
73 test::InputBuilder
rx_input(
74 "Enter receive port.", DEFAULT_RECEIVE_PORT
, 1, INT_MAX
);
75 *rx_port
= rx_input
.GetIntInput();
78 std::string
GetIpAddress(const std::string display_text
) {
79 test::InputBuilder
input(display_text
, DEFAULT_SEND_IP
, INT_MIN
, INT_MAX
);
80 std::string ip_address
= input
.GetStringInput();
81 // Ensure IP address is either the default value or in correct form.
82 while (ip_address
!= DEFAULT_SEND_IP
&&
83 std::count(ip_address
.begin(), ip_address
.end(), '.') != 3) {
84 ip_address
= input
.GetStringInput();
89 void GetSsrcs(AudioReceiverConfig
* audio_config
) {
90 test::InputBuilder
input_tx(
91 "Choose audio sender SSRC.", DEFAULT_AUDIO_FEEDBACK_SSRC
, 1, INT_MAX
);
92 audio_config
->feedback_ssrc
= input_tx
.GetIntInput();
94 test::InputBuilder
input_rx(
95 "Choose audio receiver SSRC.", DEFAULT_AUDIO_INCOMING_SSRC
, 1, INT_MAX
);
96 audio_config
->incoming_ssrc
= input_rx
.GetIntInput();
99 void GetSsrcs(VideoReceiverConfig
* video_config
) {
100 test::InputBuilder
input_tx(
101 "Choose video sender SSRC.", DEFAULT_VIDEO_FEEDBACK_SSRC
, 1, INT_MAX
);
102 video_config
->feedback_ssrc
= input_tx
.GetIntInput();
104 test::InputBuilder
input_rx(
105 "Choose video receiver SSRC.", DEFAULT_VIDEO_INCOMING_SSRC
, 1, INT_MAX
);
106 video_config
->incoming_ssrc
= input_rx
.GetIntInput();
109 #if defined(OS_LINUX)
110 void GetWindowSize(int* width
, int* height
) {
111 // Resolution values based on sender settings
112 test::InputBuilder
input_w(
113 "Choose window width.", kVideoWindowWidth
, 144, 1920);
114 *width
= input_w
.GetIntInput();
116 test::InputBuilder
input_h(
117 "Choose window height.", kVideoWindowHeight
, 176, 1080);
118 *height
= input_h
.GetIntInput();
122 void GetPayloadtype(AudioReceiverConfig
* audio_config
) {
123 test::InputBuilder
input("Choose audio receiver payload type.",
124 DEFAULT_AUDIO_PAYLOAD_TYPE
,
127 audio_config
->rtp_payload_type
= input
.GetIntInput();
130 AudioReceiverConfig
GetAudioReceiverConfig() {
131 AudioReceiverConfig audio_config
= GetDefaultAudioReceiverConfig();
132 GetSsrcs(&audio_config
);
133 GetPayloadtype(&audio_config
);
134 audio_config
.rtp_max_delay_ms
= 300;
138 void GetPayloadtype(VideoReceiverConfig
* video_config
) {
139 test::InputBuilder
input("Choose video receiver payload type.",
140 DEFAULT_VIDEO_PAYLOAD_TYPE
,
143 video_config
->rtp_payload_type
= input
.GetIntInput();
146 VideoReceiverConfig
GetVideoReceiverConfig() {
147 VideoReceiverConfig video_config
= GetDefaultVideoReceiverConfig();
148 GetSsrcs(&video_config
);
149 GetPayloadtype(&video_config
);
150 video_config
.rtp_max_delay_ms
= 300;
154 AudioParameters
ToAudioParameters(const AudioReceiverConfig
& config
) {
155 const int samples_in_10ms
= config
.frequency
/ 100;
156 return AudioParameters(AudioParameters::AUDIO_PCM_LOW_LATENCY
,
157 GuessChannelLayout(config
.channels
),
158 config
.frequency
, 32, samples_in_10ms
);
161 // An InProcessReceiver that renders video frames to a LinuxOutputWindow and
162 // audio frames via Chromium's audio stack.
164 // InProcessReceiver pushes audio and video frames to this subclass, and these
165 // frames are pushed into a queue. Then, for audio, the Chromium audio stack
166 // will make polling calls on a separate, unknown thread whereby audio frames
167 // are pulled out of the audio queue as needed. For video, however, NaivePlayer
168 // is responsible for scheduling updates to the screen itself. For both, the
169 // queues are pruned (i.e., received frames are skipped) when the system is not
170 // able to play back as fast as frames are entering the queue.
172 // This is NOT a good reference implementation for a Cast receiver player since:
173 // 1. It only skips frames to handle slower-than-expected playout, or halts
174 // playback to handle frame underruns.
175 // 2. It makes no attempt to synchronize the timing of playout of the video
176 // frames with the audio frames.
177 // 3. It does nothing to smooth or hide discontinuities in playback due to
178 // timing issues or missing frames.
179 class NaivePlayer
: public InProcessReceiver
,
180 public AudioOutputStream::AudioSourceCallback
{
182 NaivePlayer(const scoped_refptr
<CastEnvironment
>& cast_environment
,
183 const net::IPEndPoint
& local_end_point
,
184 const net::IPEndPoint
& remote_end_point
,
185 const AudioReceiverConfig
& audio_config
,
186 const VideoReceiverConfig
& video_config
,
189 : InProcessReceiver(cast_environment
,
194 // Maximum age is the duration of 3 video frames. 3 was chosen
195 // arbitrarily, but seems to work well.
196 max_frame_age_(base::TimeDelta::FromSeconds(1) * 3 /
197 video_config
.max_frame_rate
),
198 #if defined(OS_LINUX)
199 render_(0, 0, window_width
, window_height
, "Cast_receiver"),
201 num_video_frames_processed_(0),
202 num_audio_frames_processed_(0),
203 currently_playing_audio_frame_start_(-1) {}
205 virtual ~NaivePlayer() {}
207 virtual void Start() OVERRIDE
{
208 AudioManager::Get()->GetTaskRunner()->PostTask(
210 base::Bind(&NaivePlayer::StartAudioOutputOnAudioManagerThread
,
211 base::Unretained(this)));
212 // Note: No need to wait for audio polling to start since the push-and-pull
213 // mechanism is synchronized via the |audio_playout_queue_|.
214 InProcessReceiver::Start();
217 virtual void Stop() OVERRIDE
{
218 // First, stop audio output to the Chromium audio stack.
219 base::WaitableEvent
done(false, false);
220 DCHECK(!AudioManager::Get()->GetTaskRunner()->BelongsToCurrentThread());
221 AudioManager::Get()->GetTaskRunner()->PostTask(
223 base::Bind(&NaivePlayer::StopAudioOutputOnAudioManagerThread
,
224 base::Unretained(this),
228 // Now, stop receiving new frames.
229 InProcessReceiver::Stop();
231 // Finally, clear out any frames remaining in the queues.
232 while (!audio_playout_queue_
.empty()) {
233 const scoped_ptr
<AudioBus
> to_be_deleted(
234 audio_playout_queue_
.front().second
);
235 audio_playout_queue_
.pop_front();
237 video_playout_queue_
.clear();
241 void StartAudioOutputOnAudioManagerThread() {
242 DCHECK(AudioManager::Get()->GetTaskRunner()->BelongsToCurrentThread());
243 DCHECK(!audio_output_stream_
);
244 audio_output_stream_
.reset(AudioManager::Get()->MakeAudioOutputStreamProxy(
245 ToAudioParameters(audio_config()), ""));
246 if (audio_output_stream_
.get() && audio_output_stream_
->Open()) {
247 audio_output_stream_
->Start(this);
249 LOG(ERROR
) << "Failed to open an audio output stream. "
250 << "Audio playback disabled.";
251 audio_output_stream_
.reset();
255 void StopAudioOutputOnAudioManagerThread(base::WaitableEvent
* done
) {
256 DCHECK(AudioManager::Get()->GetTaskRunner()->BelongsToCurrentThread());
257 if (audio_output_stream_
.get()) {
258 audio_output_stream_
->Stop();
259 audio_output_stream_
->Close();
260 audio_output_stream_
.reset();
265 ////////////////////////////////////////////////////////////////////
266 // InProcessReceiver overrides.
268 virtual void OnVideoFrame(const scoped_refptr
<VideoFrame
>& video_frame
,
269 const base::TimeTicks
& playout_time
,
270 bool is_continuous
) OVERRIDE
{
271 DCHECK(cast_env()->CurrentlyOn(CastEnvironment::MAIN
));
272 LOG_IF(WARNING
, !is_continuous
)
273 << "Video: Discontinuity in received frames.";
274 video_playout_queue_
.push_back(std::make_pair(playout_time
, video_frame
));
275 ScheduleVideoPlayout();
277 if (media::cast::test::DecodeBarcode(video_frame
, &frame_no
)) {
278 video_play_times_
.insert(
279 std::pair
<uint16
, base::TimeTicks
>(frame_no
, playout_time
));
281 VLOG(2) << "Barcode decode failed!";
285 virtual void OnAudioFrame(scoped_ptr
<AudioBus
> audio_frame
,
286 const base::TimeTicks
& playout_time
,
287 bool is_continuous
) OVERRIDE
{
288 DCHECK(cast_env()->CurrentlyOn(CastEnvironment::MAIN
));
289 LOG_IF(WARNING
, !is_continuous
)
290 << "Audio: Discontinuity in received frames.";
291 base::AutoLock
auto_lock(audio_lock_
);
293 if (media::cast::DecodeTimestamp(audio_frame
->channel(0),
294 audio_frame
->frames(),
296 // Since there are lots of audio packets with the same frame_no,
297 // we really want to make sure that we get the playout_time from
298 // the first one. If is_continous is true, then it's possible
299 // that we already missed the first one.
300 if (is_continuous
&& frame_no
== last_audio_frame_no_
+ 1) {
301 audio_play_times_
.insert(
302 std::pair
<uint16
, base::TimeTicks
>(frame_no
, playout_time
));
304 last_audio_frame_no_
= frame_no
;
306 VLOG(2) << "Audio decode failed!";
307 last_audio_frame_no_
= -2;
309 audio_playout_queue_
.push_back(
310 std::make_pair(playout_time
, audio_frame
.release()));
313 // End of InProcessReceiver overrides.
314 ////////////////////////////////////////////////////////////////////
316 ////////////////////////////////////////////////////////////////////
317 // AudioSourceCallback implementation.
319 virtual int OnMoreData(AudioBus
* dest
, AudioBuffersState buffers_state
)
321 // Note: This method is being invoked by a separate thread unknown to us
322 // (i.e., outside of CastEnvironment).
324 int samples_remaining
= dest
->frames();
326 while (samples_remaining
> 0) {
327 // Get next audio frame ready for playout.
328 if (!currently_playing_audio_frame_
.get()) {
329 base::AutoLock
auto_lock(audio_lock_
);
331 // Prune the queue, skipping entries that are too old.
332 // TODO(miu): Use |buffers_state| to account for audio buffering delays
334 const base::TimeTicks earliest_time_to_play
=
335 cast_env()->Clock()->NowTicks() - max_frame_age_
;
336 while (!audio_playout_queue_
.empty() &&
337 audio_playout_queue_
.front().first
< earliest_time_to_play
) {
338 PopOneAudioFrame(true);
340 if (audio_playout_queue_
.empty())
343 currently_playing_audio_frame_
= PopOneAudioFrame(false).Pass();
344 currently_playing_audio_frame_start_
= 0;
347 // Copy some or all of the samples in |currently_playing_audio_frame_| to
348 // |dest|. Once all samples in |currently_playing_audio_frame_| have been
349 // consumed, release it.
350 const int num_samples_to_copy
=
351 std::min(samples_remaining
,
352 currently_playing_audio_frame_
->frames() -
353 currently_playing_audio_frame_start_
);
354 currently_playing_audio_frame_
->CopyPartialFramesTo(
355 currently_playing_audio_frame_start_
,
359 samples_remaining
-= num_samples_to_copy
;
360 currently_playing_audio_frame_start_
+= num_samples_to_copy
;
361 if (currently_playing_audio_frame_start_
==
362 currently_playing_audio_frame_
->frames()) {
363 currently_playing_audio_frame_
.reset();
367 // If |dest| has not been fully filled, then an underrun has occurred; and
368 // fill the remainder of |dest| with zeros.
369 if (samples_remaining
> 0) {
370 // Note: Only logging underruns after the first frame has been received.
371 LOG_IF(WARNING
, currently_playing_audio_frame_start_
!= -1)
372 << "Audio: Playback underrun of " << samples_remaining
<< " samples!";
373 dest
->ZeroFramesPartial(dest
->frames() - samples_remaining
,
377 return dest
->frames();
380 virtual int OnMoreIOData(AudioBus
* source
,
382 AudioBuffersState buffers_state
) OVERRIDE
{
383 return OnMoreData(dest
, buffers_state
);
386 virtual void OnError(AudioOutputStream
* stream
) OVERRIDE
{
387 LOG(ERROR
) << "AudioOutputStream reports an error. "
388 << "Playback is unlikely to continue.";
391 // End of AudioSourceCallback implementation.
392 ////////////////////////////////////////////////////////////////////
394 void ScheduleVideoPlayout() {
395 DCHECK(cast_env()->CurrentlyOn(CastEnvironment::MAIN
));
397 // Prune the queue, skipping entries that are too old.
398 const base::TimeTicks now
= cast_env()->Clock()->NowTicks();
399 const base::TimeTicks earliest_time_to_play
= now
- max_frame_age_
;
400 while (!video_playout_queue_
.empty() &&
401 video_playout_queue_
.front().first
< earliest_time_to_play
) {
402 PopOneVideoFrame(true);
405 // If the queue is not empty, schedule playout of its first frame.
406 if (video_playout_queue_
.empty()) {
407 video_playout_timer_
.Stop();
409 video_playout_timer_
.Start(
411 video_playout_queue_
.front().first
- now
,
412 base::Bind(&NaivePlayer::PlayNextVideoFrame
,
413 base::Unretained(this)));
417 void PlayNextVideoFrame() {
418 DCHECK(cast_env()->CurrentlyOn(CastEnvironment::MAIN
));
419 if (!video_playout_queue_
.empty()) {
420 const scoped_refptr
<VideoFrame
> video_frame
= PopOneVideoFrame(false);
422 render_
.RenderFrame(video_frame
);
425 ScheduleVideoPlayout();
429 scoped_refptr
<VideoFrame
> PopOneVideoFrame(bool is_being_skipped
) {
430 DCHECK(cast_env()->CurrentlyOn(CastEnvironment::MAIN
));
432 if (is_being_skipped
) {
433 VLOG(1) << "VideoFrame[" << num_video_frames_processed_
<< "]: Skipped.";
435 VLOG(1) << "VideoFrame[" << num_video_frames_processed_
<< "]: Playing "
436 << (cast_env()->Clock()->NowTicks() -
437 video_playout_queue_
.front().first
).InMicroseconds()
438 << " usec later than intended.";
441 const scoped_refptr
<VideoFrame
> ret
= video_playout_queue_
.front().second
;
442 video_playout_queue_
.pop_front();
443 ++num_video_frames_processed_
;
447 scoped_ptr
<AudioBus
> PopOneAudioFrame(bool was_skipped
) {
448 audio_lock_
.AssertAcquired();
451 VLOG(1) << "AudioFrame[" << num_audio_frames_processed_
<< "]: Skipped";
453 VLOG(1) << "AudioFrame[" << num_audio_frames_processed_
<< "]: Playing "
454 << (cast_env()->Clock()->NowTicks() -
455 audio_playout_queue_
.front().first
).InMicroseconds()
456 << " usec later than intended.";
459 scoped_ptr
<AudioBus
> ret(audio_playout_queue_
.front().second
);
460 audio_playout_queue_
.pop_front();
461 ++num_audio_frames_processed_
;
466 if (video_play_times_
.size() > 30 &&
467 audio_play_times_
.size() > 30) {
468 size_t num_events
= 0;
469 base::TimeDelta delta
;
470 std::map
<uint16
, base::TimeTicks
>::iterator audio_iter
, video_iter
;
471 for (video_iter
= video_play_times_
.begin();
472 video_iter
!= video_play_times_
.end();
474 audio_iter
= audio_play_times_
.find(video_iter
->first
);
475 if (audio_iter
!= audio_play_times_
.end()) {
477 // Positive values means audio is running behind video.
478 delta
+= audio_iter
->second
- video_iter
->second
;
482 if (num_events
> 30) {
483 VLOG(0) << "Audio behind by: "
484 << (delta
/ num_events
).InMilliseconds()
486 video_play_times_
.clear();
487 audio_play_times_
.clear();
489 } else if (video_play_times_
.size() + audio_play_times_
.size() > 500) {
490 // We are decoding audio or video timestamps, but not both, clear it out.
491 video_play_times_
.clear();
492 audio_play_times_
.clear();
496 // Frames in the queue older than this (relative to NowTicks()) will be
497 // dropped (i.e., playback is falling behind).
498 const base::TimeDelta max_frame_age_
;
500 // Outputs created, started, and destroyed by this NaivePlayer.
502 test::LinuxOutputWindow render_
;
504 scoped_ptr
<AudioOutputStream
> audio_output_stream_
;
506 // Video playout queue.
507 typedef std::pair
<base::TimeTicks
, scoped_refptr
<VideoFrame
> >
509 std::deque
<VideoQueueEntry
> video_playout_queue_
;
510 int64 num_video_frames_processed_
;
512 base::OneShotTimer
<NaivePlayer
> video_playout_timer_
;
514 // Audio playout queue, synchronized by |audio_lock_|.
515 base::Lock audio_lock_
;
516 typedef std::pair
<base::TimeTicks
, AudioBus
*> AudioQueueEntry
;
517 std::deque
<AudioQueueEntry
> audio_playout_queue_
;
518 int64 num_audio_frames_processed_
;
520 // These must only be used on the audio thread calling OnMoreData().
521 scoped_ptr
<AudioBus
> currently_playing_audio_frame_
;
522 int currently_playing_audio_frame_start_
;
524 std::map
<uint16
, base::TimeTicks
> audio_play_times_
;
525 std::map
<uint16
, base::TimeTicks
> video_play_times_
;
526 int32 last_audio_frame_no_
;
532 int main(int argc
, char** argv
) {
533 base::AtExitManager at_exit
;
534 CommandLine::Init(argc
, argv
);
535 InitLogging(logging::LoggingSettings());
537 scoped_refptr
<media::cast::CastEnvironment
> cast_environment(
538 new media::cast::StandaloneCastEnvironment
);
540 // Start up Chromium audio system.
541 media::FakeAudioLogFactory fake_audio_log_factory_
;
542 const scoped_ptr
<media::AudioManager
> audio_manager(
543 media::AudioManager::Create(&fake_audio_log_factory_
));
544 CHECK(media::AudioManager::Get());
546 media::cast::AudioReceiverConfig audio_config
=
547 media::cast::GetAudioReceiverConfig();
548 media::cast::VideoReceiverConfig video_config
=
549 media::cast::GetVideoReceiverConfig();
551 // Determine local and remote endpoints.
552 int remote_port
, local_port
;
553 media::cast::GetPorts(&remote_port
, &local_port
);
555 LOG(ERROR
) << "Invalid local port.";
558 std::string remote_ip_address
= media::cast::GetIpAddress("Enter remote IP.");
559 std::string local_ip_address
= media::cast::GetIpAddress("Enter local IP.");
560 net::IPAddressNumber remote_ip_number
;
561 net::IPAddressNumber local_ip_number
;
562 if (!net::ParseIPLiteralToNumber(remote_ip_address
, &remote_ip_number
)) {
563 LOG(ERROR
) << "Invalid remote IP address.";
566 if (!net::ParseIPLiteralToNumber(local_ip_address
, &local_ip_number
)) {
567 LOG(ERROR
) << "Invalid local IP address.";
570 net::IPEndPoint
remote_end_point(remote_ip_number
, remote_port
);
571 net::IPEndPoint
local_end_point(local_ip_number
, local_port
);
573 // Create and start the player.
574 int window_width
= 0;
575 int window_height
= 0;
576 #if defined(OS_LINUX)
577 media::cast::GetWindowSize(&window_width
, &window_height
);
579 media::cast::NaivePlayer
player(cast_environment
,
588 base::MessageLoop().Run(); // Run forever (i.e., until SIGTERM).