1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/speech/chrome_speech_recognition_manager_delegate.h"
10 #include "base/bind.h"
11 #include "base/prefs/pref_service.h"
12 #include "base/synchronization/lock.h"
13 #include "base/threading/thread_restrictions.h"
14 #include "base/utf_string_conversions.h"
15 #include "chrome/browser/browser_process.h"
16 #include "chrome/browser/extensions/extension_service.h"
17 #include "chrome/browser/profiles/profile_manager.h"
18 #include "chrome/browser/speech/chrome_speech_recognition_preferences.h"
19 #include "chrome/browser/tab_contents/tab_util.h"
20 #include "chrome/common/pref_names.h"
21 #include "content/public/browser/browser_thread.h"
22 #include "content/public/browser/notification_registrar.h"
23 #include "content/public/browser/notification_source.h"
24 #include "content/public/browser/notification_types.h"
25 #include "content/public/browser/render_process_host.h"
26 #include "content/public/browser/render_view_host.h"
27 #include "content/public/browser/resource_context.h"
28 #include "content/public/browser/speech_recognition_manager.h"
29 #include "content/public/browser/speech_recognition_session_config.h"
30 #include "content/public/browser/speech_recognition_session_context.h"
31 #include "content/public/browser/web_contents.h"
32 #include "content/public/common/speech_recognition_error.h"
33 #include "content/public/common/speech_recognition_result.h"
34 #include "extensions/browser/view_type_utils.h"
35 #include "grit/generated_resources.h"
36 #include "net/url_request/url_request_context_getter.h"
37 #include "ui/base/l10n/l10n_util.h"
40 #include "chrome/installer/util/wmi.h"
43 using content::BrowserThread
;
44 using content::SpeechRecognitionManager
;
45 using content::WebContents
;
49 const char kExtensionPrefix
[] = "chrome-extension://";
51 bool RequiresBubble(int session_id
) {
52 return SpeechRecognitionManager::GetInstance()->
53 GetSessionContext(session_id
).requested_by_page_element
;
56 bool RequiresTrayIcon(int session_id
) {
57 return !RequiresBubble(session_id
);
64 // Asynchronously fetches the PC and audio hardware/driver info if
65 // the user has opted into UMA. This information is sent with speech input
66 // requests to the server for identifying and improving quality issues with
67 // specific device configurations.
68 class ChromeSpeechRecognitionManagerDelegate::OptionalRequestInfo
69 : public base::RefCountedThreadSafe
<OptionalRequestInfo
> {
71 OptionalRequestInfo() : can_report_metrics_(false) {
75 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
76 // UMA opt-in can be checked only from the UI thread, so switch to that.
77 BrowserThread::PostTask(BrowserThread::UI
, FROM_HERE
,
78 base::Bind(&OptionalRequestInfo::CheckUMAAndGetHardwareInfo
, this));
81 void CheckUMAAndGetHardwareInfo() {
82 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI
));
83 if (g_browser_process
->local_state()->GetBoolean(
84 prefs::kMetricsReportingEnabled
)) {
85 // Access potentially slow OS calls from the FILE thread.
86 BrowserThread::PostTask(BrowserThread::FILE, FROM_HERE
,
87 base::Bind(&OptionalRequestInfo::GetHardwareInfo
, this));
91 void GetHardwareInfo() {
92 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::FILE));
93 base::AutoLock
lock(lock_
);
94 can_report_metrics_
= true;
95 string16 device_model
=
96 SpeechRecognitionManager::GetInstance()->GetAudioInputDeviceModel();
99 installer::WMIComputerSystem::GetModel() + L
"|" + device_model
);
100 #else // defined(OS_WIN)
101 value_
= UTF16ToUTF8(device_model
);
102 #endif // defined(OS_WIN)
105 std::string
value() {
106 base::AutoLock
lock(lock_
);
110 bool can_report_metrics() {
111 base::AutoLock
lock(lock_
);
112 return can_report_metrics_
;
116 friend class base::RefCountedThreadSafe
<OptionalRequestInfo
>;
118 ~OptionalRequestInfo() {}
122 bool can_report_metrics_
;
124 DISALLOW_COPY_AND_ASSIGN(OptionalRequestInfo
);
127 // Simple utility to get notified when a WebContent (a tab or an extension's
128 // background page) is closed or crashes. Both the callback site and the
129 // callback thread are passed by the caller in the constructor.
130 // There is no restriction on the constructor, however this class must be
131 // destroyed on the UI thread, due to the NotificationRegistrar dependency.
132 class ChromeSpeechRecognitionManagerDelegate::TabWatcher
133 : public base::RefCountedThreadSafe
<TabWatcher
>,
134 public content::NotificationObserver
{
136 typedef base::Callback
<void(int render_process_id
, int render_view_id
)>
139 TabWatcher(TabClosedCallback tab_closed_callback
,
140 BrowserThread::ID callback_thread
)
141 : tab_closed_callback_(tab_closed_callback
),
142 callback_thread_(callback_thread
) {
145 // Starts monitoring the WebContents corresponding to the given
146 // |render_process_id|, |render_view_id| pair, invoking |tab_closed_callback_|
147 // if closed/unloaded.
148 void Watch(int render_process_id
, int render_view_id
) {
149 if (!BrowserThread::CurrentlyOn(BrowserThread::UI
)) {
150 BrowserThread::PostTask(BrowserThread::UI
, FROM_HERE
, base::Bind(
151 &TabWatcher::Watch
, this, render_process_id
, render_view_id
));
155 WebContents
* web_contents
= tab_util::GetWebContentsByID(render_process_id
,
157 // Sessions initiated by speech input extension APIs will end up in a NULL
158 // WebContent here, but they are properly managed by the
159 // chrome::SpeechInputExtensionManager. However, sessions initiated within a
160 // extension using the (new) speech JS APIs, will be properly handled here.
161 // TODO(primiano) turn this line into a DCHECK once speech input extension
162 // API is deprecated.
166 // Avoid multiple registrations on |registrar_| for the same |web_contents|.
167 if (FindWebContents(web_contents
) != registered_web_contents_
.end()) {
170 registered_web_contents_
.push_back(
171 WebContentsInfo(web_contents
, render_process_id
, render_view_id
));
173 // Lazy initialize the registrar.
174 if (!registrar_
.get())
175 registrar_
.reset(new content::NotificationRegistrar());
177 registrar_
->Add(this,
178 content::NOTIFICATION_WEB_CONTENTS_SWAPPED
,
179 content::Source
<WebContents
>(web_contents
));
180 registrar_
->Add(this,
181 content::NOTIFICATION_WEB_CONTENTS_DISCONNECTED
,
182 content::Source
<WebContents
>(web_contents
));
185 // content::NotificationObserver implementation.
186 virtual void Observe(int type
,
187 const content::NotificationSource
& source
,
188 const content::NotificationDetails
& details
) OVERRIDE
{
189 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI
));
190 DCHECK(type
== content::NOTIFICATION_WEB_CONTENTS_DISCONNECTED
||
191 type
== content::NOTIFICATION_WEB_CONTENTS_SWAPPED
);
193 WebContents
* web_contents
= content::Source
<WebContents
>(source
).ptr();
194 std::vector
<WebContentsInfo
>::iterator iter
= FindWebContents(web_contents
);
195 DCHECK(iter
!= registered_web_contents_
.end());
196 int render_process_id
= iter
->render_process_id
;
197 int render_view_id
= iter
->render_view_id
;
198 registered_web_contents_
.erase(iter
);
200 registrar_
->Remove(this,
201 content::NOTIFICATION_WEB_CONTENTS_SWAPPED
,
202 content::Source
<WebContents
>(web_contents
));
203 registrar_
->Remove(this,
204 content::NOTIFICATION_WEB_CONTENTS_DISCONNECTED
,
205 content::Source
<WebContents
>(web_contents
));
207 BrowserThread::PostTask(callback_thread_
, FROM_HERE
, base::Bind(
208 tab_closed_callback_
, render_process_id
, render_view_id
));
212 struct WebContentsInfo
{
213 WebContentsInfo(content::WebContents
* web_contents
,
214 int render_process_id
,
216 : web_contents(web_contents
),
217 render_process_id(render_process_id
),
218 render_view_id(render_view_id
) {}
220 ~WebContentsInfo() {}
222 content::WebContents
* web_contents
;
223 int render_process_id
;
227 friend class base::RefCountedThreadSafe
<TabWatcher
>;
229 virtual ~TabWatcher() {
230 // Must be destroyed on the UI thread due to |registrar_| non thread-safety.
231 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI
));
234 // Helper function to find the iterator in |registered_web_contents_| which
235 // contains |web_contents|.
236 std::vector
<WebContentsInfo
>::iterator
FindWebContents(
237 content::WebContents
* web_contents
) {
238 for (std::vector
<WebContentsInfo
>::iterator
i(
239 registered_web_contents_
.begin());
240 i
!= registered_web_contents_
.end(); ++i
) {
241 if (i
->web_contents
== web_contents
)
245 return registered_web_contents_
.end();
248 // Lazy-initialized and used on the UI thread to handle web contents
249 // notifications (tab closing).
250 scoped_ptr
<content::NotificationRegistrar
> registrar_
;
252 // Keeps track of which WebContent(s) have been registered, in order to avoid
253 // double registrations on |registrar_| and to pass the correct render
254 // process id and render view id to |tab_closed_callback_| after the process
256 std::vector
<WebContentsInfo
> registered_web_contents_
;
258 // Callback used to notify, on the thread specified by |callback_thread_| the
259 // closure of a registered tab.
260 TabClosedCallback tab_closed_callback_
;
261 content::BrowserThread::ID callback_thread_
;
263 DISALLOW_COPY_AND_ASSIGN(TabWatcher
);
266 ChromeSpeechRecognitionManagerDelegate
267 ::ChromeSpeechRecognitionManagerDelegate() {
270 ChromeSpeechRecognitionManagerDelegate
271 ::~ChromeSpeechRecognitionManagerDelegate() {
272 if (bubble_controller_
.get())
273 bubble_controller_
->CloseBubble();
276 void ChromeSpeechRecognitionManagerDelegate::InfoBubbleButtonClicked(
277 int session_id
, SpeechRecognitionBubble::Button button
) {
278 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
280 // Note, the session might have been destroyed, therefore avoid calls to the
281 // manager which imply its existance (e.g., GetSessionContext()).
283 if (button
== SpeechRecognitionBubble::BUTTON_CANCEL
) {
284 GetBubbleController()->CloseBubble();
285 last_session_config_
.reset();
287 // We can safely call AbortSession even if the session has already ended,
288 // the manager's public methods are reliable and will handle it properly.
289 SpeechRecognitionManager::GetInstance()->AbortSession(session_id
);
290 } else if (button
== SpeechRecognitionBubble::BUTTON_TRY_AGAIN
) {
291 GetBubbleController()->CloseBubble();
292 RestartLastSession();
298 void ChromeSpeechRecognitionManagerDelegate::InfoBubbleFocusChanged(
300 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
302 // This check is needed since on some systems (MacOS), in rare cases, if the
303 // user clicks repeatedly and fast on the input element, the FocusChanged
304 // event (corresponding to the old session that should be aborted) can be
305 // received after a new session (corresponding to the 2nd click) is started.
306 if (GetBubbleController()->GetActiveSessionID() != session_id
)
309 // Note, the session might have been destroyed, therefore avoid calls to the
310 // manager which imply its existance (e.g., GetSessionContext()).
311 GetBubbleController()->CloseBubble();
312 last_session_config_
.reset();
314 // Clicking outside the bubble means we should abort.
315 SpeechRecognitionManager::GetInstance()->AbortSession(session_id
);
318 void ChromeSpeechRecognitionManagerDelegate::RestartLastSession() {
319 DCHECK(last_session_config_
.get());
320 SpeechRecognitionManager
* manager
= SpeechRecognitionManager::GetInstance();
321 const int new_session_id
= manager
->CreateSession(*last_session_config_
);
322 DCHECK_NE(SpeechRecognitionManager::kSessionIDInvalid
, new_session_id
);
323 last_session_config_
.reset();
324 manager
->StartSession(new_session_id
);
327 void ChromeSpeechRecognitionManagerDelegate::TabClosedCallback(
328 int render_process_id
, int render_view_id
) {
329 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
331 SpeechRecognitionManager
* manager
= SpeechRecognitionManager::GetInstance();
332 // |manager| becomes NULL if a browser shutdown happens between the post of
333 // this task (from the UI thread) and this call (on the IO thread). In this
334 // case we just return.
338 manager
->AbortAllSessionsForRenderView(render_process_id
, render_view_id
);
340 if (bubble_controller_
.get() &&
341 bubble_controller_
->IsShowingBubbleForRenderView(render_process_id
,
343 bubble_controller_
->CloseBubble();
347 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionStart(
349 const content::SpeechRecognitionSessionContext
& context
=
350 SpeechRecognitionManager::GetInstance()->GetSessionContext(session_id
);
352 if (RequiresBubble(session_id
)) {
353 // Copy the configuration of the session (for the "try again" button).
354 last_session_config_
.reset(new content::SpeechRecognitionSessionConfig(
355 SpeechRecognitionManager::GetInstance()->GetSessionConfig(session_id
)));
357 // Create and show the bubble.
358 GetBubbleController()->CreateBubble(session_id
,
359 context
.render_process_id
,
360 context
.render_view_id
,
361 context
.element_rect
);
364 // Register callback to auto abort session on tab closure.
365 // |tab_watcher_| is lazyly istantiated on the first call.
366 if (!tab_watcher_
.get()) {
367 tab_watcher_
= new TabWatcher(
368 base::Bind(&ChromeSpeechRecognitionManagerDelegate::TabClosedCallback
,
369 base::Unretained(this)),
372 tab_watcher_
->Watch(context
.render_process_id
, context
.render_view_id
);
375 void ChromeSpeechRecognitionManagerDelegate::OnAudioStart(int session_id
) {
376 if (RequiresBubble(session_id
)) {
377 DCHECK_EQ(session_id
, GetBubbleController()->GetActiveSessionID());
378 GetBubbleController()->SetBubbleRecordingMode();
382 void ChromeSpeechRecognitionManagerDelegate::OnEnvironmentEstimationComplete(
386 void ChromeSpeechRecognitionManagerDelegate::OnSoundStart(int session_id
) {
389 void ChromeSpeechRecognitionManagerDelegate::OnSoundEnd(int session_id
) {
392 void ChromeSpeechRecognitionManagerDelegate::OnAudioEnd(int session_id
) {
393 // OnAudioEnd can be also raised after an abort, when the bubble has already
395 if (GetBubbleController()->GetActiveSessionID() == session_id
) {
396 DCHECK(RequiresBubble(session_id
));
397 GetBubbleController()->SetBubbleRecognizingMode();
401 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionResults(
402 int session_id
, const content::SpeechRecognitionResults
& result
) {
403 // The bubble will be closed upon the OnEnd event, which will follow soon.
406 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionError(
407 int session_id
, const content::SpeechRecognitionError
& error
) {
408 // An error can be dispatched when the bubble is not visible anymore.
409 if (GetBubbleController()->GetActiveSessionID() != session_id
)
411 DCHECK(RequiresBubble(session_id
));
413 int error_message_id
= 0;
414 switch (error
.code
) {
415 case content::SPEECH_RECOGNITION_ERROR_AUDIO
:
416 switch (error
.details
) {
417 case content::SPEECH_AUDIO_ERROR_DETAILS_NO_MIC
:
418 error_message_id
= IDS_SPEECH_INPUT_NO_MIC
;
420 case content::SPEECH_AUDIO_ERROR_DETAILS_IN_USE
:
421 error_message_id
= IDS_SPEECH_INPUT_MIC_IN_USE
;
424 error_message_id
= IDS_SPEECH_INPUT_MIC_ERROR
;
428 case content::SPEECH_RECOGNITION_ERROR_ABORTED
:
429 error_message_id
= IDS_SPEECH_INPUT_ABORTED
;
431 case content::SPEECH_RECOGNITION_ERROR_NO_SPEECH
:
432 error_message_id
= IDS_SPEECH_INPUT_NO_SPEECH
;
434 case content::SPEECH_RECOGNITION_ERROR_NO_MATCH
:
435 error_message_id
= IDS_SPEECH_INPUT_NO_RESULTS
;
437 case content::SPEECH_RECOGNITION_ERROR_NETWORK
:
438 error_message_id
= IDS_SPEECH_INPUT_NET_ERROR
;
441 NOTREACHED() << "unknown error " << error
.code
;
444 GetBubbleController()->SetBubbleMessage(
445 l10n_util::GetStringUTF16(error_message_id
));
448 void ChromeSpeechRecognitionManagerDelegate::OnAudioLevelsChange(
449 int session_id
, float volume
, float noise_volume
) {
450 if (GetBubbleController()->GetActiveSessionID() == session_id
) {
451 DCHECK(RequiresBubble(session_id
));
452 GetBubbleController()->SetBubbleInputVolume(volume
, noise_volume
);
456 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionEnd(int session_id
) {
457 // The only case in which the OnRecognitionEnd should not close the bubble is
458 // when we are showing an error. In this case the bubble will be closed by
459 // the |InfoBubbleFocusChanged| method, when the users clicks either the
460 // "Cancel" button or outside of the bubble.
461 if (GetBubbleController()->GetActiveSessionID() == session_id
&&
462 !GetBubbleController()->IsShowingMessage()) {
463 DCHECK(RequiresBubble(session_id
));
464 GetBubbleController()->CloseBubble();
468 void ChromeSpeechRecognitionManagerDelegate::GetDiagnosticInformation(
469 bool* can_report_metrics
,
470 std::string
* hardware_info
) {
471 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
472 if (!optional_request_info_
.get()) {
473 optional_request_info_
= new OptionalRequestInfo();
474 // Since hardware info is optional with speech input requests, we start an
475 // asynchronous fetch here and move on with recording audio. This first
476 // speech input request would send an empty string for hardware info and
477 // subsequent requests may have the hardware info available if the fetch
478 // completed before them. This way we don't end up stalling the user with
479 // a long wait and disk seeks when they click on a UI element and start
481 optional_request_info_
->Refresh();
483 *can_report_metrics
= optional_request_info_
->can_report_metrics();
484 *hardware_info
= optional_request_info_
->value();
487 void ChromeSpeechRecognitionManagerDelegate::CheckRecognitionIsAllowed(
489 base::Callback
<void(bool ask_user
, bool is_allowed
)> callback
) {
490 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
492 const content::SpeechRecognitionSessionContext
& context
=
493 SpeechRecognitionManager::GetInstance()->GetSessionContext(session_id
);
495 // Make sure that initiators (extensions/web pages) properly set the
496 // |render_process_id| field, which is needed later to retrieve the
497 // ChromeSpeechRecognitionPreferences associated to their profile.
498 DCHECK_NE(context
.render_process_id
, 0);
500 // Check that the render view type is appropriate, and whether or not we
501 // need to request permission from the user.
502 BrowserThread::PostTask(BrowserThread::UI
, FROM_HERE
,
503 base::Bind(&CheckRenderViewType
,
505 context
.render_process_id
,
506 context
.render_view_id
,
507 RequiresTrayIcon(session_id
)));
510 content::SpeechRecognitionEventListener
*
511 ChromeSpeechRecognitionManagerDelegate::GetEventListener() {
515 void ChromeSpeechRecognitionManagerDelegate::CheckRenderViewType(
516 base::Callback
<void(bool ask_user
, bool is_allowed
)> callback
,
517 int render_process_id
,
520 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI
));
521 const content::RenderViewHost
* render_view_host
=
522 content::RenderViewHost::FromID(render_process_id
, render_view_id
);
524 bool allowed
= false;
525 bool ask_permission
= false;
527 if (!render_view_host
) {
529 // If there is no render view, we cannot show the speech bubble, so this
532 ask_permission
= false;
534 // This happens for extensions. Manifest should be checked for permission.
536 ask_permission
= false;
538 BrowserThread::PostTask(BrowserThread::IO
, FROM_HERE
,
539 base::Bind(callback
, ask_permission
, allowed
));
543 WebContents
* web_contents
= WebContents::FromRenderViewHost(render_view_host
);
544 extensions::ViewType view_type
= extensions::GetViewType(web_contents
);
546 // TODO(kalman): Also enable speech bubble for extension popups
547 // (VIEW_TYPE_EXTENSION_POPUP) once popup-like control UI works properly in
548 // extensions: http://crbug.com/163851.
549 // Right now the extension popup closes and dismisses immediately on user
551 if (view_type
== extensions::VIEW_TYPE_TAB_CONTENTS
||
552 view_type
== extensions::VIEW_TYPE_APP_SHELL
) {
553 // If it is a tab, we can show the speech input bubble or ask for
558 ask_permission
= true;
561 BrowserThread::PostTask(BrowserThread::IO
, FROM_HERE
,
562 base::Bind(callback
, ask_permission
, allowed
));
565 SpeechRecognitionBubbleController
*
566 ChromeSpeechRecognitionManagerDelegate::GetBubbleController() {
567 if (!bubble_controller_
.get())
568 bubble_controller_
= new SpeechRecognitionBubbleController(this);
569 return bubble_controller_
.get();
572 } // namespace speech