1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/media/media_internals.h"
7 #include "base/metrics/histogram.h"
8 #include "base/strings/string16.h"
9 #include "base/strings/string_number_conversions.h"
10 #include "base/strings/stringprintf.h"
11 #include "content/public/browser/browser_thread.h"
12 #include "content/public/browser/notification_observer.h"
13 #include "content/public/browser/notification_registrar.h"
14 #include "content/public/browser/notification_service.h"
15 #include "content/public/browser/notification_types.h"
16 #include "content/public/browser/render_frame_host.h"
17 #include "content/public/browser/render_process_host.h"
18 #include "content/public/browser/web_contents.h"
19 #include "content/public/browser/web_ui.h"
20 #include "media/audio/audio_parameters.h"
21 #include "media/base/media_log_event.h"
22 #include "media/filters/decrypting_video_decoder.h"
23 #include "media/filters/gpu_video_decoder.h"
27 static base::LazyInstance
<content::MediaInternals
>::Leaky g_media_internals
=
28 LAZY_INSTANCE_INITIALIZER
;
30 base::string16
SerializeUpdate(const std::string
& function
,
31 const base::Value
* value
) {
32 return content::WebUI::GetJavascriptCall(
33 function
, std::vector
<const base::Value
*>(1, value
));
36 std::string
EffectsToString(int effects
) {
37 if (effects
== media::AudioParameters::NO_EFFECTS
)
44 { media::AudioParameters::ECHO_CANCELLER
, "ECHO_CANCELLER" },
45 { media::AudioParameters::DUCKING
, "DUCKING" },
46 { media::AudioParameters::KEYBOARD_MIC
, "KEYBOARD_MIC" },
47 { media::AudioParameters::HOTWORD
, "HOTWORD" },
51 for (size_t i
= 0; i
< arraysize(flags
); ++i
) {
52 if (effects
& flags
[i
].flag
) {
56 effects
&= ~flags
[i
].flag
;
63 ret
+= base::IntToString(effects
);
69 std::string
FormatToString(media::AudioParameters::Format format
) {
71 case media::AudioParameters::AUDIO_PCM_LINEAR
:
73 case media::AudioParameters::AUDIO_PCM_LOW_LATENCY
:
74 return "pcm_low_latency";
75 case media::AudioParameters::AUDIO_FAKE
:
77 case media::AudioParameters::AUDIO_LAST_FORMAT
:
85 const char kAudioLogStatusKey
[] = "status";
86 const char kAudioLogUpdateFunction
[] = "media.updateAudioComponent";
92 class AudioLogImpl
: public media::AudioLog
{
94 AudioLogImpl(int owner_id
,
95 media::AudioLogFactory::AudioComponent component
,
96 content::MediaInternals
* media_internals
);
97 ~AudioLogImpl() override
;
99 void OnCreated(int component_id
,
100 const media::AudioParameters
& params
,
101 const std::string
& device_id
) override
;
102 void OnStarted(int component_id
) override
;
103 void OnStopped(int component_id
) override
;
104 void OnClosed(int component_id
) override
;
105 void OnError(int component_id
) override
;
106 void OnSetVolume(int component_id
, double volume
) override
;
108 // Called by MediaInternals to update the WebContents title for a stream.
109 void SendWebContentsTitle(int component_id
,
110 int render_process_id
,
111 int render_frame_id
);
114 void SendSingleStringUpdate(int component_id
,
115 const std::string
& key
,
116 const std::string
& value
);
117 void StoreComponentMetadata(int component_id
, base::DictionaryValue
* dict
);
118 std::string
FormatCacheKey(int component_id
);
120 static void SendWebContentsTitleHelper(const std::string
& cache_key
,
121 scoped_ptr
<base::DictionaryValue
> dict
,
122 int render_process_id
,
123 int render_frame_id
);
126 const media::AudioLogFactory::AudioComponent component_
;
127 content::MediaInternals
* const media_internals_
;
129 DISALLOW_COPY_AND_ASSIGN(AudioLogImpl
);
132 AudioLogImpl::AudioLogImpl(int owner_id
,
133 media::AudioLogFactory::AudioComponent component
,
134 content::MediaInternals
* media_internals
)
135 : owner_id_(owner_id
),
136 component_(component
),
137 media_internals_(media_internals
) {}
139 AudioLogImpl::~AudioLogImpl() {}
141 void AudioLogImpl::OnCreated(int component_id
,
142 const media::AudioParameters
& params
,
143 const std::string
& device_id
) {
144 base::DictionaryValue dict
;
145 StoreComponentMetadata(component_id
, &dict
);
147 dict
.SetString(kAudioLogStatusKey
, "created");
148 dict
.SetString("device_id", device_id
);
149 dict
.SetString("device_type", FormatToString(params
.format()));
150 dict
.SetInteger("frames_per_buffer", params
.frames_per_buffer());
151 dict
.SetInteger("sample_rate", params
.sample_rate());
152 dict
.SetInteger("channels", params
.channels());
153 dict
.SetString("channel_layout",
154 ChannelLayoutToString(params
.channel_layout()));
155 dict
.SetString("effects", EffectsToString(params
.effects()));
157 media_internals_
->SendAudioLogUpdate(MediaInternals::CREATE
,
158 FormatCacheKey(component_id
),
159 kAudioLogUpdateFunction
, &dict
);
162 void AudioLogImpl::OnStarted(int component_id
) {
163 SendSingleStringUpdate(component_id
, kAudioLogStatusKey
, "started");
166 void AudioLogImpl::OnStopped(int component_id
) {
167 SendSingleStringUpdate(component_id
, kAudioLogStatusKey
, "stopped");
170 void AudioLogImpl::OnClosed(int component_id
) {
171 base::DictionaryValue dict
;
172 StoreComponentMetadata(component_id
, &dict
);
173 dict
.SetString(kAudioLogStatusKey
, "closed");
174 media_internals_
->SendAudioLogUpdate(MediaInternals::UPDATE_AND_DELETE
,
175 FormatCacheKey(component_id
),
176 kAudioLogUpdateFunction
, &dict
);
179 void AudioLogImpl::OnError(int component_id
) {
180 SendSingleStringUpdate(component_id
, "error_occurred", "true");
183 void AudioLogImpl::OnSetVolume(int component_id
, double volume
) {
184 base::DictionaryValue dict
;
185 StoreComponentMetadata(component_id
, &dict
);
186 dict
.SetDouble("volume", volume
);
187 media_internals_
->SendAudioLogUpdate(MediaInternals::UPDATE_IF_EXISTS
,
188 FormatCacheKey(component_id
),
189 kAudioLogUpdateFunction
, &dict
);
192 void AudioLogImpl::SendWebContentsTitle(int component_id
,
193 int render_process_id
,
194 int render_frame_id
) {
195 scoped_ptr
<base::DictionaryValue
> dict(new base::DictionaryValue());
196 StoreComponentMetadata(component_id
, dict
.get());
197 SendWebContentsTitleHelper(FormatCacheKey(component_id
), dict
.Pass(),
198 render_process_id
, render_frame_id
);
201 std::string
AudioLogImpl::FormatCacheKey(int component_id
) {
202 return base::StringPrintf("%d:%d:%d", owner_id_
, component_
, component_id
);
206 void AudioLogImpl::SendWebContentsTitleHelper(
207 const std::string
& cache_key
,
208 scoped_ptr
<base::DictionaryValue
> dict
,
209 int render_process_id
,
210 int render_frame_id
) {
211 // Page title information can only be retrieved from the UI thread.
212 if (!BrowserThread::CurrentlyOn(BrowserThread::UI
)) {
213 BrowserThread::PostTask(
214 BrowserThread::UI
, FROM_HERE
,
215 base::Bind(&SendWebContentsTitleHelper
, cache_key
, base::Passed(&dict
),
216 render_process_id
, render_frame_id
));
220 const WebContents
* web_contents
= WebContents::FromRenderFrameHost(
221 RenderFrameHost::FromID(render_process_id
, render_frame_id
));
225 // Note: by this point the given audio log entry could have been destroyed, so
226 // we use UPDATE_IF_EXISTS to discard such instances.
227 dict
->SetInteger("render_process_id", render_process_id
);
228 dict
->SetString("web_contents_title", web_contents
->GetTitle());
229 MediaInternals::GetInstance()->SendAudioLogUpdate(
230 MediaInternals::UPDATE_IF_EXISTS
, cache_key
, kAudioLogUpdateFunction
,
234 void AudioLogImpl::SendSingleStringUpdate(int component_id
,
235 const std::string
& key
,
236 const std::string
& value
) {
237 base::DictionaryValue dict
;
238 StoreComponentMetadata(component_id
, &dict
);
239 dict
.SetString(key
, value
);
240 media_internals_
->SendAudioLogUpdate(MediaInternals::UPDATE_IF_EXISTS
,
241 FormatCacheKey(component_id
),
242 kAudioLogUpdateFunction
, &dict
);
245 void AudioLogImpl::StoreComponentMetadata(int component_id
,
246 base::DictionaryValue
* dict
) {
247 dict
->SetInteger("owner_id", owner_id_
);
248 dict
->SetInteger("component_id", component_id
);
249 dict
->SetInteger("component_type", component_
);
252 class MediaInternals::MediaInternalsUMAHandler
: public NotificationObserver
{
254 MediaInternalsUMAHandler();
256 // NotificationObserver implementation.
257 void Observe(int type
,
258 const NotificationSource
& source
,
259 const NotificationDetails
& details
) override
;
261 // Reports the pipeline status to UMA for every player
262 // associated with the renderer process and then deletes the player state.
263 void LogAndClearPlayersInRenderer(int render_process_id
);
265 // Helper function to save the event payload to RendererPlayerMap.
266 void SavePlayerState(const media::MediaLogEvent
& event
,
267 int render_process_id
);
270 struct PipelineInfo
{
271 media::PipelineStatus last_pipeline_status
;
275 bool video_decoder_changed
;
276 std::string audio_codec_name
;
277 std::string video_codec_name
;
278 std::string video_decoder
;
280 : last_pipeline_status(media::PIPELINE_OK
),
284 video_decoder_changed(false) {}
287 // Helper function to report PipelineStatus associated with a player to UMA.
288 void ReportUMAForPipelineStatus(const PipelineInfo
& player_info
);
290 // Helper to generate PipelineStatus UMA name for AudioVideo streams.
291 std::string
GetUMANameForAVStream(const PipelineInfo
& player_info
);
294 typedef std::map
<int, PipelineInfo
> PlayerInfoMap
;
296 // Key is renderer id
297 typedef std::map
<int, PlayerInfoMap
> RendererPlayerMap
;
299 // Stores player information per renderer
300 RendererPlayerMap renderer_info_
;
302 NotificationRegistrar registrar_
;
304 DISALLOW_COPY_AND_ASSIGN(MediaInternalsUMAHandler
);
307 MediaInternals::MediaInternalsUMAHandler::MediaInternalsUMAHandler() {
308 registrar_
.Add(this, NOTIFICATION_RENDERER_PROCESS_TERMINATED
,
309 NotificationService::AllBrowserContextsAndSources());
312 void MediaInternals::MediaInternalsUMAHandler::Observe(
314 const NotificationSource
& source
,
315 const NotificationDetails
& details
) {
316 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
317 DCHECK_EQ(type
, NOTIFICATION_RENDERER_PROCESS_TERMINATED
);
318 RenderProcessHost
* process
= Source
<RenderProcessHost
>(source
).ptr();
320 // Post the task to the IO thread to avoid race in updating renderer_info_ map
321 // by both SavePlayerState & LogAndClearPlayersInRenderer from different
323 // Using base::Unretained() on MediaInternalsUMAHandler is safe since
324 // it is owned by MediaInternals and share the same lifetime
325 BrowserThread::PostTask(
326 BrowserThread::IO
, FROM_HERE
,
327 base::Bind(&MediaInternalsUMAHandler::LogAndClearPlayersInRenderer
,
328 base::Unretained(this), process
->GetID()));
331 void MediaInternals::MediaInternalsUMAHandler::SavePlayerState(
332 const media::MediaLogEvent
& event
,
333 int render_process_id
) {
334 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
335 PlayerInfoMap
& player_info
= renderer_info_
[render_process_id
];
336 switch (event
.type
) {
337 case media::MediaLogEvent::PIPELINE_ERROR
: {
339 event
.params
.GetInteger("pipeline_error", &status
);
340 player_info
[event
.id
].last_pipeline_status
=
341 static_cast<media::PipelineStatus
>(status
);
344 case media::MediaLogEvent::PROPERTY_CHANGE
:
345 if (event
.params
.HasKey("found_audio_stream")) {
346 event
.params
.GetBoolean("found_audio_stream",
347 &player_info
[event
.id
].has_audio
);
349 if (event
.params
.HasKey("found_video_stream")) {
350 event
.params
.GetBoolean("found_video_stream",
351 &player_info
[event
.id
].has_video
);
353 if (event
.params
.HasKey("audio_codec_name")) {
354 event
.params
.GetString("audio_codec_name",
355 &player_info
[event
.id
].audio_codec_name
);
357 if (event
.params
.HasKey("video_codec_name")) {
358 event
.params
.GetString("video_codec_name",
359 &player_info
[event
.id
].video_codec_name
);
361 if (event
.params
.HasKey("video_decoder")) {
362 std::string
previous_video_decoder(player_info
[event
.id
].video_decoder
);
363 event
.params
.GetString("video_decoder",
364 &player_info
[event
.id
].video_decoder
);
365 if (!previous_video_decoder
.empty() &&
366 previous_video_decoder
!= player_info
[event
.id
].video_decoder
) {
367 player_info
[event
.id
].video_decoder_changed
= true;
370 if (event
.params
.HasKey("video_dds")) {
371 event
.params
.GetBoolean("video_dds", &player_info
[event
.id
].video_dds
);
380 std::string
MediaInternals::MediaInternalsUMAHandler::GetUMANameForAVStream(
381 const PipelineInfo
& player_info
) {
382 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
383 static const char kPipelineUmaPrefix
[] = "Media.PipelineStatus.AudioVideo.";
384 std::string uma_name
= kPipelineUmaPrefix
;
385 if (player_info
.video_codec_name
== "vp8") {
387 } else if (player_info
.video_codec_name
== "vp9") {
389 } else if (player_info
.video_codec_name
== "h264") {
392 return uma_name
+ "Other";
395 if (player_info
.video_decoder
==
396 media::DecryptingVideoDecoder::kDecoderName
) {
397 return uma_name
+ "DVD";
400 if (player_info
.video_dds
) {
404 if (player_info
.video_decoder
== media::GpuVideoDecoder::kDecoderName
) {
412 void MediaInternals::MediaInternalsUMAHandler::ReportUMAForPipelineStatus(
413 const PipelineInfo
& player_info
) {
414 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
415 if (player_info
.has_video
&& player_info
.has_audio
) {
416 base::LinearHistogram::FactoryGet(
417 GetUMANameForAVStream(player_info
), 1, media::PIPELINE_STATUS_MAX
,
418 media::PIPELINE_STATUS_MAX
+ 1,
419 base::HistogramBase::kUmaTargetedHistogramFlag
)
420 ->Add(player_info
.last_pipeline_status
);
421 } else if (player_info
.has_audio
) {
422 UMA_HISTOGRAM_ENUMERATION("Media.PipelineStatus.AudioOnly",
423 player_info
.last_pipeline_status
,
424 media::PIPELINE_STATUS_MAX
+ 1);
425 } else if (player_info
.has_video
) {
426 UMA_HISTOGRAM_ENUMERATION("Media.PipelineStatus.VideoOnly",
427 player_info
.last_pipeline_status
,
428 media::PIPELINE_STATUS_MAX
+ 1);
430 UMA_HISTOGRAM_ENUMERATION("Media.PipelineStatus.Unsupported",
431 player_info
.last_pipeline_status
,
432 media::PIPELINE_STATUS_MAX
+ 1);
434 // Report whether video decoder fallback happened, but only if a video decoder
436 if (!player_info
.video_decoder
.empty()) {
437 UMA_HISTOGRAM_BOOLEAN("Media.VideoDecoderFallback",
438 player_info
.video_decoder_changed
);
442 void MediaInternals::MediaInternalsUMAHandler::LogAndClearPlayersInRenderer(
443 int render_process_id
) {
444 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
445 auto players_it
= renderer_info_
.find(render_process_id
);
446 if (players_it
== renderer_info_
.end())
448 auto it
= players_it
->second
.begin();
449 while (it
!= players_it
->second
.end()) {
450 ReportUMAForPipelineStatus(it
->second
);
451 players_it
->second
.erase(it
++);
455 MediaInternals
* MediaInternals::GetInstance() {
456 return g_media_internals
.Pointer();
459 MediaInternals::MediaInternals()
460 : owner_ids_(), uma_handler_(new MediaInternalsUMAHandler()) {
463 MediaInternals::~MediaInternals() {}
465 void MediaInternals::OnMediaEvents(
466 int render_process_id
, const std::vector
<media::MediaLogEvent
>& events
) {
467 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
468 // Notify observers that |event| has occurred.
469 for (auto event
= events
.begin(); event
!= events
.end(); ++event
) {
470 base::DictionaryValue dict
;
471 dict
.SetInteger("renderer", render_process_id
);
472 dict
.SetInteger("player", event
->id
);
473 dict
.SetString("type", media::MediaLog::EventTypeToString(event
->type
));
475 // TODO(dalecurtis): This is technically not correct. TimeTicks "can't" be
476 // converted to to a human readable time format. See base/time/time.h.
477 const double ticks
= event
->time
.ToInternalValue();
478 const double ticks_millis
= ticks
/ base::Time::kMicrosecondsPerMillisecond
;
479 dict
.SetDouble("ticksMillis", ticks_millis
);
481 // Convert PipelineStatus to human readable string
482 if (event
->type
== media::MediaLogEvent::PIPELINE_ERROR
) {
484 if (!event
->params
.GetInteger("pipeline_error", &status
) ||
485 status
< static_cast<int>(media::PIPELINE_OK
) ||
486 status
> static_cast<int>(media::PIPELINE_STATUS_MAX
)) {
489 media::PipelineStatus error
= static_cast<media::PipelineStatus
>(status
);
490 dict
.SetString("params.pipeline_error",
491 media::MediaLog::PipelineStatusToString(error
));
493 dict
.Set("params", event
->params
.DeepCopy());
496 SendUpdate(SerializeUpdate("media.onMediaEvent", &dict
));
497 uma_handler_
->SavePlayerState(*event
, render_process_id
);
501 void MediaInternals::AddUpdateCallback(const UpdateCallback
& callback
) {
502 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
503 update_callbacks_
.push_back(callback
);
506 void MediaInternals::RemoveUpdateCallback(const UpdateCallback
& callback
) {
507 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
508 for (size_t i
= 0; i
< update_callbacks_
.size(); ++i
) {
509 if (update_callbacks_
[i
].Equals(callback
)) {
510 update_callbacks_
.erase(update_callbacks_
.begin() + i
);
517 void MediaInternals::SendAudioStreamData() {
518 base::string16 audio_stream_update
;
520 base::AutoLock
auto_lock(lock_
);
521 audio_stream_update
= SerializeUpdate(
522 "media.onReceiveAudioStreamData", &audio_streams_cached_data_
);
524 SendUpdate(audio_stream_update
);
527 void MediaInternals::SendVideoCaptureDeviceCapabilities() {
528 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
529 SendUpdate(SerializeUpdate("media.onReceiveVideoCaptureCapabilities",
530 &video_capture_capabilities_cached_data_
));
533 void MediaInternals::UpdateVideoCaptureDeviceCapabilities(
534 const media::VideoCaptureDeviceInfos
& video_capture_device_infos
) {
535 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
536 video_capture_capabilities_cached_data_
.Clear();
538 for (const auto& video_capture_device_info
: video_capture_device_infos
) {
539 base::ListValue
* format_list
= new base::ListValue();
540 for (const auto& format
: video_capture_device_info
.supported_formats
)
541 format_list
->AppendString(format
.ToString());
543 base::DictionaryValue
* device_dict
= new base::DictionaryValue();
544 device_dict
->SetString("id", video_capture_device_info
.name
.id());
545 device_dict
->SetString(
546 "name", video_capture_device_info
.name
.GetNameAndModel());
547 device_dict
->Set("formats", format_list
);
548 #if defined(OS_WIN) || defined(OS_MACOSX) || defined(OS_LINUX) || \
550 device_dict
->SetString(
551 "captureApi", video_capture_device_info
.name
.GetCaptureApiTypeString());
553 video_capture_capabilities_cached_data_
.Append(device_dict
);
556 if (update_callbacks_
.size() > 0)
557 SendVideoCaptureDeviceCapabilities();
560 scoped_ptr
<media::AudioLog
> MediaInternals::CreateAudioLog(
561 AudioComponent component
) {
562 base::AutoLock
auto_lock(lock_
);
563 return scoped_ptr
<media::AudioLog
>(new AudioLogImpl(
564 owner_ids_
[component
]++, component
, this));
567 void MediaInternals::SetWebContentsTitleForAudioLogEntry(
569 int render_process_id
,
571 media::AudioLog
* audio_log
) {
572 static_cast<AudioLogImpl
*>(audio_log
)
573 ->SendWebContentsTitle(component_id
, render_process_id
, render_frame_id
);
576 void MediaInternals::SendUpdate(const base::string16
& update
) {
577 // SendUpdate() may be called from any thread, but must run on the IO thread.
578 // TODO(dalecurtis): This is pretty silly since the update callbacks simply
579 // forward the calls to the UI thread. We should avoid the extra hop.
580 if (!BrowserThread::CurrentlyOn(BrowserThread::IO
)) {
581 BrowserThread::PostTask(BrowserThread::IO
, FROM_HERE
, base::Bind(
582 &MediaInternals::SendUpdate
, base::Unretained(this), update
));
586 for (size_t i
= 0; i
< update_callbacks_
.size(); i
++)
587 update_callbacks_
[i
].Run(update
);
590 void MediaInternals::SendAudioLogUpdate(AudioLogUpdateType type
,
591 const std::string
& cache_key
,
592 const std::string
& function
,
593 const base::DictionaryValue
* value
) {
595 base::AutoLock
auto_lock(lock_
);
596 const bool has_entry
= audio_streams_cached_data_
.HasKey(cache_key
);
597 if ((type
== UPDATE_IF_EXISTS
|| type
== UPDATE_AND_DELETE
) && !has_entry
) {
599 } else if (!has_entry
) {
600 DCHECK_EQ(type
, CREATE
);
601 audio_streams_cached_data_
.Set(cache_key
, value
->DeepCopy());
602 } else if (type
== UPDATE_AND_DELETE
) {
603 scoped_ptr
<base::Value
> out_value
;
604 CHECK(audio_streams_cached_data_
.Remove(cache_key
, &out_value
));
606 base::DictionaryValue
* existing_dict
= NULL
;
608 audio_streams_cached_data_
.GetDictionary(cache_key
, &existing_dict
));
609 existing_dict
->MergeDictionary(value
);
613 SendUpdate(SerializeUpdate(function
, value
));
616 } // namespace content