1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/speech/chrome_speech_recognition_manager_delegate.h"
10 #include "base/bind.h"
11 #include "base/synchronization/lock.h"
12 #include "base/threading/thread_restrictions.h"
13 #include "base/utf_string_conversions.h"
14 #include "chrome/browser/browser_process.h"
15 #include "chrome/browser/extensions/extension_service.h"
16 #include "chrome/browser/prefs/pref_service.h"
17 #include "chrome/browser/profiles/profile_manager.h"
18 #include "chrome/browser/speech/chrome_speech_recognition_preferences.h"
19 #include "chrome/browser/speech/speech_recognition_tray_icon_controller.h"
20 #include "chrome/browser/tab_contents/tab_util.h"
21 #include "chrome/browser/view_type_utils.h"
22 #include "chrome/common/pref_names.h"
23 #include "content/public/browser/browser_thread.h"
24 #include "content/public/browser/notification_registrar.h"
25 #include "content/public/browser/notification_source.h"
26 #include "content/public/browser/notification_types.h"
27 #include "content/public/browser/render_process_host.h"
28 #include "content/public/browser/render_view_host.h"
29 #include "content/public/browser/resource_context.h"
30 #include "content/public/browser/speech_recognition_manager.h"
31 #include "content/public/browser/speech_recognition_session_config.h"
32 #include "content/public/browser/speech_recognition_session_context.h"
33 #include "content/public/browser/web_contents.h"
34 #include "content/public/common/speech_recognition_error.h"
35 #include "content/public/common/speech_recognition_result.h"
36 #include "grit/generated_resources.h"
37 #include "net/url_request/url_request_context_getter.h"
38 #include "ui/base/l10n/l10n_util.h"
41 #include "chrome/installer/util/wmi.h"
44 using content::BrowserThread
;
45 using content::SpeechRecognitionManager
;
46 using content::WebContents
;
50 const char kExtensionPrefix
[] = "chrome-extension://";
52 bool RequiresBubble(int session_id
) {
53 return SpeechRecognitionManager::GetInstance()->
54 GetSessionContext(session_id
).requested_by_page_element
;
57 bool RequiresTrayIcon(int session_id
) {
58 return !RequiresBubble(session_id
);
65 // Asynchronously fetches the PC and audio hardware/driver info if
66 // the user has opted into UMA. This information is sent with speech input
67 // requests to the server for identifying and improving quality issues with
68 // specific device configurations.
69 class ChromeSpeechRecognitionManagerDelegate::OptionalRequestInfo
70 : public base::RefCountedThreadSafe
<OptionalRequestInfo
> {
72 OptionalRequestInfo() : can_report_metrics_(false) {
76 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
77 // UMA opt-in can be checked only from the UI thread, so switch to that.
78 BrowserThread::PostTask(BrowserThread::UI
, FROM_HERE
,
79 base::Bind(&OptionalRequestInfo::CheckUMAAndGetHardwareInfo
, this));
82 void CheckUMAAndGetHardwareInfo() {
83 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI
));
84 if (g_browser_process
->local_state()->GetBoolean(
85 prefs::kMetricsReportingEnabled
)) {
86 // Access potentially slow OS calls from the FILE thread.
87 BrowserThread::PostTask(BrowserThread::FILE, FROM_HERE
,
88 base::Bind(&OptionalRequestInfo::GetHardwareInfo
, this));
92 void GetHardwareInfo() {
93 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::FILE));
94 base::AutoLock
lock(lock_
);
95 can_report_metrics_
= true;
96 string16 device_model
=
97 SpeechRecognitionManager::GetInstance()->GetAudioInputDeviceModel();
100 installer::WMIComputerSystem::GetModel() + L
"|" + device_model
);
101 #else // defined(OS_WIN)
102 value_
= UTF16ToUTF8(device_model
);
103 #endif // defined(OS_WIN)
106 std::string
value() {
107 base::AutoLock
lock(lock_
);
111 bool can_report_metrics() {
112 base::AutoLock
lock(lock_
);
113 return can_report_metrics_
;
117 friend class base::RefCountedThreadSafe
<OptionalRequestInfo
>;
119 ~OptionalRequestInfo() {}
123 bool can_report_metrics_
;
125 DISALLOW_COPY_AND_ASSIGN(OptionalRequestInfo
);
128 // Simple utility to get notified when a WebContent (a tab or an extension's
129 // background page) is closed or crashes. Both the callback site and the
130 // callback thread are passed by the caller in the constructor.
131 // There is no restriction on the constructor, however this class must be
132 // destroyed on the UI thread, due to the NotificationRegistrar dependency.
133 class ChromeSpeechRecognitionManagerDelegate::TabWatcher
134 : public base::RefCountedThreadSafe
<TabWatcher
>,
135 public content::NotificationObserver
{
137 typedef base::Callback
<void(int render_process_id
, int render_view_id
)>
140 TabWatcher(TabClosedCallback tab_closed_callback
,
141 BrowserThread::ID callback_thread
)
142 : tab_closed_callback_(tab_closed_callback
),
143 callback_thread_(callback_thread
) {
146 // Starts monitoring the WebContents corresponding to the given
147 // |render_process_id|, |render_view_id| pair, invoking |tab_closed_callback_|
148 // if closed/unloaded.
149 void Watch(int render_process_id
, int render_view_id
) {
150 if (!BrowserThread::CurrentlyOn(BrowserThread::UI
)) {
151 BrowserThread::PostTask(BrowserThread::UI
, FROM_HERE
, base::Bind(
152 &TabWatcher::Watch
, this, render_process_id
, render_view_id
));
156 WebContents
* web_contents
= tab_util::GetWebContentsByID(render_process_id
,
158 // Sessions initiated by speech input extension APIs will end up in a NULL
159 // WebContent here, but they are properly managed by the
160 // chrome::SpeechInputExtensionManager. However, sessions initiated within a
161 // extension using the (new) speech JS APIs, will be properly handled here.
162 // TODO(primiano) turn this line into a DCHECK once speech input extension
163 // API is deprecated.
167 // Avoid multiple registrations on |registrar_| for the same |web_contents|.
168 if (registered_web_contents_
.find(web_contents
) !=
169 registered_web_contents_
.end()) {
172 registered_web_contents_
.insert(web_contents
);
174 // Lazy initialize the registrar.
175 if (!registrar_
.get())
176 registrar_
.reset(new content::NotificationRegistrar());
178 registrar_
->Add(this,
179 content::NOTIFICATION_WEB_CONTENTS_DISCONNECTED
,
180 content::Source
<WebContents
>(web_contents
));
183 // content::NotificationObserver implementation.
184 virtual void Observe(int type
,
185 const content::NotificationSource
& source
,
186 const content::NotificationDetails
& details
) OVERRIDE
{
187 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI
));
188 DCHECK_EQ(content::NOTIFICATION_WEB_CONTENTS_DISCONNECTED
, type
);
190 WebContents
* web_contents
= content::Source
<WebContents
>(source
).ptr();
191 int render_process_id
= web_contents
->GetRenderProcessHost()->GetID();
192 int render_view_id
= web_contents
->GetRenderViewHost()->GetRoutingID();
194 registrar_
->Remove(this,
195 content::NOTIFICATION_WEB_CONTENTS_DISCONNECTED
,
196 content::Source
<WebContents
>(web_contents
));
197 registered_web_contents_
.erase(web_contents
);
199 BrowserThread::PostTask(callback_thread_
, FROM_HERE
, base::Bind(
200 tab_closed_callback_
, render_process_id
, render_view_id
));
204 friend class base::RefCountedThreadSafe
<TabWatcher
>;
206 virtual ~TabWatcher() {
207 // Must be destroyed on the UI thread due to |registrar_| non thread-safety.
208 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI
));
211 // Lazy-initialized and used on the UI thread to handle web contents
212 // notifications (tab closing).
213 scoped_ptr
<content::NotificationRegistrar
> registrar_
;
215 // Keeps track of which WebContent(s) have been registered, in order to avoid
216 // double registrations on |registrar_|
217 std::set
<content::WebContents
*> registered_web_contents_
;
219 // Callback used to notify, on the thread specified by |callback_thread_| the
220 // closure of a registered tab.
221 TabClosedCallback tab_closed_callback_
;
222 content::BrowserThread::ID callback_thread_
;
224 DISALLOW_COPY_AND_ASSIGN(TabWatcher
);
227 ChromeSpeechRecognitionManagerDelegate
228 ::ChromeSpeechRecognitionManagerDelegate() {
231 ChromeSpeechRecognitionManagerDelegate
232 ::~ChromeSpeechRecognitionManagerDelegate() {
233 if (tray_icon_controller_
.get())
234 tray_icon_controller_
->Hide();
235 if (bubble_controller_
.get())
236 bubble_controller_
->CloseBubble();
239 void ChromeSpeechRecognitionManagerDelegate::InfoBubbleButtonClicked(
240 int session_id
, SpeechRecognitionBubble::Button button
) {
241 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
243 // Note, the session might have been destroyed, therefore avoid calls to the
244 // manager which imply its existance (e.g., GetSessionContext()).
246 if (button
== SpeechRecognitionBubble::BUTTON_CANCEL
) {
247 GetBubbleController()->CloseBubble();
248 last_session_config_
.reset();
250 // We can safely call AbortSession even if the session has already ended,
251 // the manager's public methods are reliable and will handle it properly.
252 SpeechRecognitionManager::GetInstance()->AbortSession(session_id
);
253 } else if (button
== SpeechRecognitionBubble::BUTTON_TRY_AGAIN
) {
254 GetBubbleController()->CloseBubble();
255 RestartLastSession();
261 void ChromeSpeechRecognitionManagerDelegate::InfoBubbleFocusChanged(
263 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
265 // This check is needed since on some systems (MacOS), in rare cases, if the
266 // user clicks repeatedly and fast on the input element, the FocusChanged
267 // event (corresponding to the old session that should be aborted) can be
268 // received after a new session (corresponding to the 2nd click) is started.
269 if (GetBubbleController()->GetActiveSessionID() != session_id
)
272 // Note, the session might have been destroyed, therefore avoid calls to the
273 // manager which imply its existance (e.g., GetSessionContext()).
274 GetBubbleController()->CloseBubble();
275 last_session_config_
.reset();
277 // Clicking outside the bubble means we should abort.
278 SpeechRecognitionManager::GetInstance()->AbortSession(session_id
);
281 void ChromeSpeechRecognitionManagerDelegate::RestartLastSession() {
282 DCHECK(last_session_config_
.get());
283 SpeechRecognitionManager
* manager
= SpeechRecognitionManager::GetInstance();
284 const int new_session_id
= manager
->CreateSession(*last_session_config_
);
285 DCHECK_NE(SpeechRecognitionManager::kSessionIDInvalid
, new_session_id
);
286 last_session_config_
.reset();
287 manager
->StartSession(new_session_id
);
290 void ChromeSpeechRecognitionManagerDelegate::TabClosedCallback(
291 int render_process_id
, int render_view_id
) {
292 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
294 SpeechRecognitionManager
* manager
= SpeechRecognitionManager::GetInstance();
295 // |manager| becomes NULL if a browser shutdown happens between the post of
296 // this task (from the UI thread) and this call (on the IO thread). In this
297 // case we just return.
301 manager
->AbortAllSessionsForRenderView(render_process_id
, render_view_id
);
303 if (bubble_controller_
.get() &&
304 bubble_controller_
->IsShowingBubbleForRenderView(render_process_id
,
306 bubble_controller_
->CloseBubble();
310 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionStart(
312 const content::SpeechRecognitionSessionContext
& context
=
313 SpeechRecognitionManager::GetInstance()->GetSessionContext(session_id
);
315 if (RequiresBubble(session_id
)) {
316 // Copy the configuration of the session (for the "try again" button).
317 last_session_config_
.reset(new content::SpeechRecognitionSessionConfig(
318 SpeechRecognitionManager::GetInstance()->GetSessionConfig(session_id
)));
320 // Create and show the bubble.
321 GetBubbleController()->CreateBubble(session_id
,
322 context
.render_process_id
,
323 context
.render_view_id
,
324 context
.element_rect
);
327 // Register callback to auto abort session on tab closure.
328 // |tab_watcher_| is lazyly istantiated on the first call.
329 if (!tab_watcher_
.get()) {
330 tab_watcher_
= new TabWatcher(
331 base::Bind(&ChromeSpeechRecognitionManagerDelegate::TabClosedCallback
,
332 base::Unretained(this)),
335 tab_watcher_
->Watch(context
.render_process_id
, context
.render_view_id
);
338 void ChromeSpeechRecognitionManagerDelegate::OnAudioStart(int session_id
) {
339 if (RequiresBubble(session_id
)) {
340 DCHECK_EQ(session_id
, GetBubbleController()->GetActiveSessionID());
341 GetBubbleController()->SetBubbleRecordingMode();
342 } else if (RequiresTrayIcon(session_id
)) {
343 // We post the action to the UI thread for sessions requiring a tray icon,
344 // since ChromeSpeechRecognitionPreferences (which requires UI thread) is
345 // involved for determining whether a security alert balloon is required.
346 const content::SpeechRecognitionSessionContext
& context
=
347 SpeechRecognitionManager::GetInstance()->GetSessionContext(session_id
);
348 BrowserThread::PostTask(BrowserThread::UI
, FROM_HERE
, base::Bind(
349 &ChromeSpeechRecognitionManagerDelegate::ShowTrayIconOnUIThread
,
350 context
.context_name
,
351 context
.render_process_id
,
352 scoped_refptr
<SpeechRecognitionTrayIconController
>(
353 GetTrayIconController())));
357 void ChromeSpeechRecognitionManagerDelegate::OnEnvironmentEstimationComplete(
361 void ChromeSpeechRecognitionManagerDelegate::OnSoundStart(int session_id
) {
364 void ChromeSpeechRecognitionManagerDelegate::OnSoundEnd(int session_id
) {
367 void ChromeSpeechRecognitionManagerDelegate::OnAudioEnd(int session_id
) {
368 // OnAudioEnd can be also raised after an abort, when the bubble has already
370 if (GetBubbleController()->GetActiveSessionID() == session_id
) {
371 DCHECK(RequiresBubble(session_id
));
372 GetBubbleController()->SetBubbleRecognizingMode();
373 } else if (RequiresTrayIcon(session_id
)) {
374 GetTrayIconController()->Hide();
378 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionResult(
379 int session_id
, const content::SpeechRecognitionResult
& result
) {
380 // The bubble will be closed upon the OnEnd event, which will follow soon.
383 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionError(
384 int session_id
, const content::SpeechRecognitionError
& error
) {
385 // An error can be dispatched when the bubble is not visible anymore.
386 if (GetBubbleController()->GetActiveSessionID() != session_id
)
388 DCHECK(RequiresBubble(session_id
));
390 int error_message_id
= 0;
391 switch (error
.code
) {
392 case content::SPEECH_RECOGNITION_ERROR_AUDIO
:
393 switch (error
.details
) {
394 case content::SPEECH_AUDIO_ERROR_DETAILS_NO_MIC
:
395 error_message_id
= IDS_SPEECH_INPUT_NO_MIC
;
397 case content::SPEECH_AUDIO_ERROR_DETAILS_IN_USE
:
398 error_message_id
= IDS_SPEECH_INPUT_MIC_IN_USE
;
401 error_message_id
= IDS_SPEECH_INPUT_MIC_ERROR
;
405 case content::SPEECH_RECOGNITION_ERROR_ABORTED
:
406 error_message_id
= IDS_SPEECH_INPUT_ABORTED
;
408 case content::SPEECH_RECOGNITION_ERROR_NO_SPEECH
:
409 error_message_id
= IDS_SPEECH_INPUT_NO_SPEECH
;
411 case content::SPEECH_RECOGNITION_ERROR_NO_MATCH
:
412 error_message_id
= IDS_SPEECH_INPUT_NO_RESULTS
;
414 case content::SPEECH_RECOGNITION_ERROR_NETWORK
:
415 error_message_id
= IDS_SPEECH_INPUT_NET_ERROR
;
418 NOTREACHED() << "unknown error " << error
.code
;
421 GetBubbleController()->SetBubbleMessage(
422 l10n_util::GetStringUTF16(error_message_id
));
425 void ChromeSpeechRecognitionManagerDelegate::OnAudioLevelsChange(
426 int session_id
, float volume
, float noise_volume
) {
427 if (GetBubbleController()->GetActiveSessionID() == session_id
) {
428 DCHECK(RequiresBubble(session_id
));
429 GetBubbleController()->SetBubbleInputVolume(volume
, noise_volume
);
430 } else if (RequiresTrayIcon(session_id
)) {
431 GetTrayIconController()->SetVUMeterVolume(volume
);
435 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionEnd(int session_id
) {
436 // The only case in which the OnRecognitionEnd should not close the bubble is
437 // when we are showing an error. In this case the bubble will be closed by
438 // the |InfoBubbleFocusChanged| method, when the users clicks either the
439 // "Cancel" button or outside of the bubble.
440 if (GetBubbleController()->GetActiveSessionID() == session_id
&&
441 !GetBubbleController()->IsShowingMessage()) {
442 DCHECK(RequiresBubble(session_id
));
443 GetBubbleController()->CloseBubble();
447 void ChromeSpeechRecognitionManagerDelegate::GetDiagnosticInformation(
448 bool* can_report_metrics
,
449 std::string
* hardware_info
) {
450 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
451 if (!optional_request_info_
.get()) {
452 optional_request_info_
= new OptionalRequestInfo();
453 // Since hardware info is optional with speech input requests, we start an
454 // asynchronous fetch here and move on with recording audio. This first
455 // speech input request would send an empty string for hardware info and
456 // subsequent requests may have the hardware info available if the fetch
457 // completed before them. This way we don't end up stalling the user with
458 // a long wait and disk seeks when they click on a UI element and start
460 optional_request_info_
->Refresh();
462 *can_report_metrics
= optional_request_info_
->can_report_metrics();
463 *hardware_info
= optional_request_info_
->value();
466 void ChromeSpeechRecognitionManagerDelegate::CheckRecognitionIsAllowed(
468 base::Callback
<void(bool ask_user
, bool is_allowed
)> callback
) {
469 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
471 const content::SpeechRecognitionSessionContext
& context
=
472 SpeechRecognitionManager::GetInstance()->GetSessionContext(session_id
);
474 // Make sure that initiators (extensions/web pages) properly set the
475 // |render_process_id| field, which is needed later to retrieve the
476 // ChromeSpeechRecognitionPreferences associated to their profile.
477 DCHECK_NE(context
.render_process_id
, 0);
479 // Check that the render view type is appropriate, and whether or not we
480 // need to request permission from the user.
481 BrowserThread::PostTask(BrowserThread::UI
, FROM_HERE
,
482 base::Bind(&CheckRenderViewType
,
484 context
.render_process_id
,
485 context
.render_view_id
,
486 RequiresTrayIcon(session_id
)));
489 content::SpeechRecognitionEventListener
*
490 ChromeSpeechRecognitionManagerDelegate::GetEventListener() {
494 void ChromeSpeechRecognitionManagerDelegate::ShowTrayIconOnUIThread(
495 const std::string
& context_name
,
496 int render_process_id
,
497 scoped_refptr
<SpeechRecognitionTrayIconController
> tray_icon_controller
) {
498 content::RenderProcessHost
* render_process_host
=
499 content::RenderProcessHost::FromID(render_process_id
);
500 DCHECK(render_process_host
);
501 content::BrowserContext
* browser_context
=
502 render_process_host
->GetBrowserContext();
503 Profile
* profile
= Profile::FromBrowserContext(browser_context
);
504 scoped_refptr
<ChromeSpeechRecognitionPreferences
> pref
=
505 ChromeSpeechRecognitionPreferences::GetForProfile(profile
);
506 bool show_notification
= pref
->ShouldShowSecurityNotification(context_name
);
507 if (show_notification
)
508 pref
->SetHasShownSecurityNotification(context_name
);
510 // Speech recognitions initiated by JS APIs within an extension (so NOT by
511 // extension API) will come with a context_name like "chrome-extension://id"
512 // (that is, their origin as injected by WebKit). In such cases we try to
513 // lookup the extension name, in order to show a more user-friendly balloon.
514 string16 initiator_name
= UTF8ToUTF16(context_name
);
515 if (context_name
.find(kExtensionPrefix
) == 0) {
516 const std::string extension_id
=
517 context_name
.substr(sizeof(kExtensionPrefix
) - 1);
518 const extensions::Extension
* extension
=
519 profile
->GetExtensionService()->GetExtensionById(extension_id
, true);
521 initiator_name
= UTF8ToUTF16(extension
->name());
524 tray_icon_controller
->Show(initiator_name
, show_notification
);
527 void ChromeSpeechRecognitionManagerDelegate::CheckRenderViewType(
528 base::Callback
<void(bool ask_user
, bool is_allowed
)> callback
,
529 int render_process_id
,
532 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI
));
533 const content::RenderViewHost
* render_view_host
=
534 content::RenderViewHost::FromID(render_process_id
, render_view_id
);
536 bool allowed
= false;
537 bool ask_permission
= false;
539 if (!render_view_host
) {
541 // If there is no render view, we cannot show the speech bubble, so this
544 ask_permission
= false;
546 // This happens for extensions. Manifest should be checked for permission.
548 ask_permission
= false;
550 BrowserThread::PostTask(BrowserThread::IO
, FROM_HERE
,
551 base::Bind(callback
, ask_permission
, allowed
));
555 WebContents
* web_contents
= WebContents::FromRenderViewHost(render_view_host
);
556 chrome::ViewType view_type
= chrome::GetViewType(web_contents
);
558 if (view_type
== chrome::VIEW_TYPE_TAB_CONTENTS
) {
559 // If it is a tab, we can show the speech input bubble or ask for
564 ask_permission
= true;
567 BrowserThread::PostTask(BrowserThread::IO
, FROM_HERE
,
568 base::Bind(callback
, ask_permission
, allowed
));
571 SpeechRecognitionBubbleController
*
572 ChromeSpeechRecognitionManagerDelegate::GetBubbleController() {
573 if (!bubble_controller_
.get())
574 bubble_controller_
= new SpeechRecognitionBubbleController(this);
575 return bubble_controller_
.get();
578 SpeechRecognitionTrayIconController
*
579 ChromeSpeechRecognitionManagerDelegate::GetTrayIconController() {
580 if (!tray_icon_controller_
.get())
581 tray_icon_controller_
= new SpeechRecognitionTrayIconController();
582 return tray_icon_controller_
.get();
586 } // namespace speech