Fix build break
[chromium-blink-merge.git] / chrome / browser / speech / chrome_speech_recognition_manager_delegate.cc
blob02726bec76e417ba5f34d91767db20a8073144cb
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/speech/chrome_speech_recognition_manager_delegate.h"
7 #include <set>
8 #include <string>
10 #include "base/bind.h"
11 #include "base/prefs/pref_service.h"
12 #include "base/synchronization/lock.h"
13 #include "base/threading/thread_restrictions.h"
14 #include "base/utf_string_conversions.h"
15 #include "chrome/browser/browser_process.h"
16 #include "chrome/browser/extensions/extension_service.h"
17 #include "chrome/browser/profiles/profile_manager.h"
18 #include "chrome/browser/speech/chrome_speech_recognition_preferences.h"
19 #include "chrome/browser/tab_contents/tab_util.h"
20 #include "chrome/common/pref_names.h"
21 #include "content/public/browser/browser_thread.h"
22 #include "content/public/browser/notification_registrar.h"
23 #include "content/public/browser/notification_source.h"
24 #include "content/public/browser/notification_types.h"
25 #include "content/public/browser/render_process_host.h"
26 #include "content/public/browser/render_view_host.h"
27 #include "content/public/browser/resource_context.h"
28 #include "content/public/browser/speech_recognition_manager.h"
29 #include "content/public/browser/speech_recognition_session_config.h"
30 #include "content/public/browser/speech_recognition_session_context.h"
31 #include "content/public/browser/web_contents.h"
32 #include "content/public/common/speech_recognition_error.h"
33 #include "content/public/common/speech_recognition_result.h"
34 #include "extensions/browser/view_type_utils.h"
35 #include "grit/generated_resources.h"
36 #include "net/url_request/url_request_context_getter.h"
37 #include "ui/base/l10n/l10n_util.h"
39 #if defined(OS_WIN)
40 #include "chrome/installer/util/wmi.h"
41 #endif
43 using content::BrowserThread;
44 using content::SpeechRecognitionManager;
45 using content::WebContents;
47 namespace {
49 const char kExtensionPrefix[] = "chrome-extension://";
51 bool RequiresBubble(int session_id) {
52 return SpeechRecognitionManager::GetInstance()->
53 GetSessionContext(session_id).requested_by_page_element;
56 bool RequiresTrayIcon(int session_id) {
57 return !RequiresBubble(session_id);
60 } // namespace
62 namespace speech {
64 // Asynchronously fetches the PC and audio hardware/driver info if
65 // the user has opted into UMA. This information is sent with speech input
66 // requests to the server for identifying and improving quality issues with
67 // specific device configurations.
68 class ChromeSpeechRecognitionManagerDelegate::OptionalRequestInfo
69 : public base::RefCountedThreadSafe<OptionalRequestInfo> {
70 public:
71 OptionalRequestInfo() : can_report_metrics_(false) {
74 void Refresh() {
75 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
76 // UMA opt-in can be checked only from the UI thread, so switch to that.
77 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE,
78 base::Bind(&OptionalRequestInfo::CheckUMAAndGetHardwareInfo, this));
81 void CheckUMAAndGetHardwareInfo() {
82 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
83 if (g_browser_process->local_state()->GetBoolean(
84 prefs::kMetricsReportingEnabled)) {
85 // Access potentially slow OS calls from the FILE thread.
86 BrowserThread::PostTask(BrowserThread::FILE, FROM_HERE,
87 base::Bind(&OptionalRequestInfo::GetHardwareInfo, this));
91 void GetHardwareInfo() {
92 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::FILE));
93 base::AutoLock lock(lock_);
94 can_report_metrics_ = true;
95 string16 device_model =
96 SpeechRecognitionManager::GetInstance()->GetAudioInputDeviceModel();
97 #if defined(OS_WIN)
98 value_ = UTF16ToUTF8(
99 installer::WMIComputerSystem::GetModel() + L"|" + device_model);
100 #else // defined(OS_WIN)
101 value_ = UTF16ToUTF8(device_model);
102 #endif // defined(OS_WIN)
105 std::string value() {
106 base::AutoLock lock(lock_);
107 return value_;
110 bool can_report_metrics() {
111 base::AutoLock lock(lock_);
112 return can_report_metrics_;
115 private:
116 friend class base::RefCountedThreadSafe<OptionalRequestInfo>;
118 ~OptionalRequestInfo() {}
120 base::Lock lock_;
121 std::string value_;
122 bool can_report_metrics_;
124 DISALLOW_COPY_AND_ASSIGN(OptionalRequestInfo);
127 // Simple utility to get notified when a WebContent (a tab or an extension's
128 // background page) is closed or crashes. Both the callback site and the
129 // callback thread are passed by the caller in the constructor.
130 // There is no restriction on the constructor, however this class must be
131 // destroyed on the UI thread, due to the NotificationRegistrar dependency.
132 class ChromeSpeechRecognitionManagerDelegate::TabWatcher
133 : public base::RefCountedThreadSafe<TabWatcher>,
134 public content::NotificationObserver {
135 public:
136 typedef base::Callback<void(int render_process_id, int render_view_id)>
137 TabClosedCallback;
139 TabWatcher(TabClosedCallback tab_closed_callback,
140 BrowserThread::ID callback_thread)
141 : tab_closed_callback_(tab_closed_callback),
142 callback_thread_(callback_thread) {
145 // Starts monitoring the WebContents corresponding to the given
146 // |render_process_id|, |render_view_id| pair, invoking |tab_closed_callback_|
147 // if closed/unloaded.
148 void Watch(int render_process_id, int render_view_id) {
149 if (!BrowserThread::CurrentlyOn(BrowserThread::UI)) {
150 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
151 &TabWatcher::Watch, this, render_process_id, render_view_id));
152 return;
155 WebContents* web_contents = tab_util::GetWebContentsByID(render_process_id,
156 render_view_id);
157 // Sessions initiated by speech input extension APIs will end up in a NULL
158 // WebContent here, but they are properly managed by the
159 // chrome::SpeechInputExtensionManager. However, sessions initiated within a
160 // extension using the (new) speech JS APIs, will be properly handled here.
161 // TODO(primiano) turn this line into a DCHECK once speech input extension
162 // API is deprecated.
163 if (!web_contents)
164 return;
166 // Avoid multiple registrations on |registrar_| for the same |web_contents|.
167 if (FindWebContents(web_contents) != registered_web_contents_.end()) {
168 return;
170 registered_web_contents_.push_back(
171 WebContentsInfo(web_contents, render_process_id, render_view_id));
173 // Lazy initialize the registrar.
174 if (!registrar_.get())
175 registrar_.reset(new content::NotificationRegistrar());
177 registrar_->Add(this,
178 content::NOTIFICATION_WEB_CONTENTS_SWAPPED,
179 content::Source<WebContents>(web_contents));
180 registrar_->Add(this,
181 content::NOTIFICATION_WEB_CONTENTS_DISCONNECTED,
182 content::Source<WebContents>(web_contents));
185 // content::NotificationObserver implementation.
186 virtual void Observe(int type,
187 const content::NotificationSource& source,
188 const content::NotificationDetails& details) OVERRIDE {
189 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
190 DCHECK(type == content::NOTIFICATION_WEB_CONTENTS_DISCONNECTED ||
191 type == content::NOTIFICATION_WEB_CONTENTS_SWAPPED);
193 WebContents* web_contents = content::Source<WebContents>(source).ptr();
194 std::vector<WebContentsInfo>::iterator iter = FindWebContents(web_contents);
195 DCHECK(iter != registered_web_contents_.end());
196 int render_process_id = iter->render_process_id;
197 int render_view_id = iter->render_view_id;
198 registered_web_contents_.erase(iter);
200 registrar_->Remove(this,
201 content::NOTIFICATION_WEB_CONTENTS_SWAPPED,
202 content::Source<WebContents>(web_contents));
203 registrar_->Remove(this,
204 content::NOTIFICATION_WEB_CONTENTS_DISCONNECTED,
205 content::Source<WebContents>(web_contents));
207 BrowserThread::PostTask(callback_thread_, FROM_HERE, base::Bind(
208 tab_closed_callback_, render_process_id, render_view_id));
211 private:
212 struct WebContentsInfo {
213 WebContentsInfo(content::WebContents* web_contents,
214 int render_process_id,
215 int render_view_id)
216 : web_contents(web_contents),
217 render_process_id(render_process_id),
218 render_view_id(render_view_id) {}
220 ~WebContentsInfo() {}
222 content::WebContents* web_contents;
223 int render_process_id;
224 int render_view_id;
227 friend class base::RefCountedThreadSafe<TabWatcher>;
229 virtual ~TabWatcher() {
230 // Must be destroyed on the UI thread due to |registrar_| non thread-safety.
231 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
234 // Helper function to find the iterator in |registered_web_contents_| which
235 // contains |web_contents|.
236 std::vector<WebContentsInfo>::iterator FindWebContents(
237 content::WebContents* web_contents) {
238 for (std::vector<WebContentsInfo>::iterator i(
239 registered_web_contents_.begin());
240 i != registered_web_contents_.end(); ++i) {
241 if (i->web_contents == web_contents)
242 return i;
245 return registered_web_contents_.end();
248 // Lazy-initialized and used on the UI thread to handle web contents
249 // notifications (tab closing).
250 scoped_ptr<content::NotificationRegistrar> registrar_;
252 // Keeps track of which WebContent(s) have been registered, in order to avoid
253 // double registrations on |registrar_| and to pass the correct render
254 // process id and render view id to |tab_closed_callback_| after the process
255 // has gone away.
256 std::vector<WebContentsInfo> registered_web_contents_;
258 // Callback used to notify, on the thread specified by |callback_thread_| the
259 // closure of a registered tab.
260 TabClosedCallback tab_closed_callback_;
261 content::BrowserThread::ID callback_thread_;
263 DISALLOW_COPY_AND_ASSIGN(TabWatcher);
266 ChromeSpeechRecognitionManagerDelegate
267 ::ChromeSpeechRecognitionManagerDelegate() {
270 ChromeSpeechRecognitionManagerDelegate
271 ::~ChromeSpeechRecognitionManagerDelegate() {
272 if (bubble_controller_.get())
273 bubble_controller_->CloseBubble();
276 void ChromeSpeechRecognitionManagerDelegate::InfoBubbleButtonClicked(
277 int session_id, SpeechRecognitionBubble::Button button) {
278 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
280 // Note, the session might have been destroyed, therefore avoid calls to the
281 // manager which imply its existance (e.g., GetSessionContext()).
283 if (button == SpeechRecognitionBubble::BUTTON_CANCEL) {
284 GetBubbleController()->CloseBubble();
285 last_session_config_.reset();
287 // We can safely call AbortSession even if the session has already ended,
288 // the manager's public methods are reliable and will handle it properly.
289 SpeechRecognitionManager::GetInstance()->AbortSession(session_id);
290 } else if (button == SpeechRecognitionBubble::BUTTON_TRY_AGAIN) {
291 GetBubbleController()->CloseBubble();
292 RestartLastSession();
293 } else {
294 NOTREACHED();
298 void ChromeSpeechRecognitionManagerDelegate::InfoBubbleFocusChanged(
299 int session_id) {
300 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
302 // This check is needed since on some systems (MacOS), in rare cases, if the
303 // user clicks repeatedly and fast on the input element, the FocusChanged
304 // event (corresponding to the old session that should be aborted) can be
305 // received after a new session (corresponding to the 2nd click) is started.
306 if (GetBubbleController()->GetActiveSessionID() != session_id)
307 return;
309 // Note, the session might have been destroyed, therefore avoid calls to the
310 // manager which imply its existance (e.g., GetSessionContext()).
311 GetBubbleController()->CloseBubble();
312 last_session_config_.reset();
314 // Clicking outside the bubble means we should abort.
315 SpeechRecognitionManager::GetInstance()->AbortSession(session_id);
318 void ChromeSpeechRecognitionManagerDelegate::RestartLastSession() {
319 DCHECK(last_session_config_.get());
320 SpeechRecognitionManager* manager = SpeechRecognitionManager::GetInstance();
321 const int new_session_id = manager->CreateSession(*last_session_config_);
322 DCHECK_NE(SpeechRecognitionManager::kSessionIDInvalid, new_session_id);
323 last_session_config_.reset();
324 manager->StartSession(new_session_id);
327 void ChromeSpeechRecognitionManagerDelegate::TabClosedCallback(
328 int render_process_id, int render_view_id) {
329 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
331 SpeechRecognitionManager* manager = SpeechRecognitionManager::GetInstance();
332 // |manager| becomes NULL if a browser shutdown happens between the post of
333 // this task (from the UI thread) and this call (on the IO thread). In this
334 // case we just return.
335 if (!manager)
336 return;
338 manager->AbortAllSessionsForRenderView(render_process_id, render_view_id);
340 if (bubble_controller_.get() &&
341 bubble_controller_->IsShowingBubbleForRenderView(render_process_id,
342 render_view_id)) {
343 bubble_controller_->CloseBubble();
347 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionStart(
348 int session_id) {
349 const content::SpeechRecognitionSessionContext& context =
350 SpeechRecognitionManager::GetInstance()->GetSessionContext(session_id);
352 if (RequiresBubble(session_id)) {
353 // Copy the configuration of the session (for the "try again" button).
354 last_session_config_.reset(new content::SpeechRecognitionSessionConfig(
355 SpeechRecognitionManager::GetInstance()->GetSessionConfig(session_id)));
357 // Create and show the bubble.
358 GetBubbleController()->CreateBubble(session_id,
359 context.render_process_id,
360 context.render_view_id,
361 context.element_rect);
364 // Register callback to auto abort session on tab closure.
365 // |tab_watcher_| is lazyly istantiated on the first call.
366 if (!tab_watcher_.get()) {
367 tab_watcher_ = new TabWatcher(
368 base::Bind(&ChromeSpeechRecognitionManagerDelegate::TabClosedCallback,
369 base::Unretained(this)),
370 BrowserThread::IO);
372 tab_watcher_->Watch(context.render_process_id, context.render_view_id);
375 void ChromeSpeechRecognitionManagerDelegate::OnAudioStart(int session_id) {
376 if (RequiresBubble(session_id)) {
377 DCHECK_EQ(session_id, GetBubbleController()->GetActiveSessionID());
378 GetBubbleController()->SetBubbleRecordingMode();
382 void ChromeSpeechRecognitionManagerDelegate::OnEnvironmentEstimationComplete(
383 int session_id) {
386 void ChromeSpeechRecognitionManagerDelegate::OnSoundStart(int session_id) {
389 void ChromeSpeechRecognitionManagerDelegate::OnSoundEnd(int session_id) {
392 void ChromeSpeechRecognitionManagerDelegate::OnAudioEnd(int session_id) {
393 // OnAudioEnd can be also raised after an abort, when the bubble has already
394 // been closed.
395 if (GetBubbleController()->GetActiveSessionID() == session_id) {
396 DCHECK(RequiresBubble(session_id));
397 GetBubbleController()->SetBubbleRecognizingMode();
401 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionResults(
402 int session_id, const content::SpeechRecognitionResults& result) {
403 // The bubble will be closed upon the OnEnd event, which will follow soon.
406 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionError(
407 int session_id, const content::SpeechRecognitionError& error) {
408 // An error can be dispatched when the bubble is not visible anymore.
409 if (GetBubbleController()->GetActiveSessionID() != session_id)
410 return;
411 DCHECK(RequiresBubble(session_id));
413 int error_message_id = 0;
414 switch (error.code) {
415 case content::SPEECH_RECOGNITION_ERROR_AUDIO:
416 switch (error.details) {
417 case content::SPEECH_AUDIO_ERROR_DETAILS_NO_MIC:
418 error_message_id = IDS_SPEECH_INPUT_NO_MIC;
419 break;
420 case content::SPEECH_AUDIO_ERROR_DETAILS_IN_USE:
421 error_message_id = IDS_SPEECH_INPUT_MIC_IN_USE;
422 break;
423 default:
424 error_message_id = IDS_SPEECH_INPUT_MIC_ERROR;
425 break;
427 break;
428 case content::SPEECH_RECOGNITION_ERROR_ABORTED:
429 error_message_id = IDS_SPEECH_INPUT_ABORTED;
430 break;
431 case content::SPEECH_RECOGNITION_ERROR_NO_SPEECH:
432 error_message_id = IDS_SPEECH_INPUT_NO_SPEECH;
433 break;
434 case content::SPEECH_RECOGNITION_ERROR_NO_MATCH:
435 error_message_id = IDS_SPEECH_INPUT_NO_RESULTS;
436 break;
437 case content::SPEECH_RECOGNITION_ERROR_NETWORK:
438 error_message_id = IDS_SPEECH_INPUT_NET_ERROR;
439 break;
440 default:
441 NOTREACHED() << "unknown error " << error.code;
442 return;
444 GetBubbleController()->SetBubbleMessage(
445 l10n_util::GetStringUTF16(error_message_id));
448 void ChromeSpeechRecognitionManagerDelegate::OnAudioLevelsChange(
449 int session_id, float volume, float noise_volume) {
450 if (GetBubbleController()->GetActiveSessionID() == session_id) {
451 DCHECK(RequiresBubble(session_id));
452 GetBubbleController()->SetBubbleInputVolume(volume, noise_volume);
456 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionEnd(int session_id) {
457 // The only case in which the OnRecognitionEnd should not close the bubble is
458 // when we are showing an error. In this case the bubble will be closed by
459 // the |InfoBubbleFocusChanged| method, when the users clicks either the
460 // "Cancel" button or outside of the bubble.
461 if (GetBubbleController()->GetActiveSessionID() == session_id &&
462 !GetBubbleController()->IsShowingMessage()) {
463 DCHECK(RequiresBubble(session_id));
464 GetBubbleController()->CloseBubble();
468 void ChromeSpeechRecognitionManagerDelegate::GetDiagnosticInformation(
469 bool* can_report_metrics,
470 std::string* hardware_info) {
471 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
472 if (!optional_request_info_.get()) {
473 optional_request_info_ = new OptionalRequestInfo();
474 // Since hardware info is optional with speech input requests, we start an
475 // asynchronous fetch here and move on with recording audio. This first
476 // speech input request would send an empty string for hardware info and
477 // subsequent requests may have the hardware info available if the fetch
478 // completed before them. This way we don't end up stalling the user with
479 // a long wait and disk seeks when they click on a UI element and start
480 // speaking.
481 optional_request_info_->Refresh();
483 *can_report_metrics = optional_request_info_->can_report_metrics();
484 *hardware_info = optional_request_info_->value();
487 void ChromeSpeechRecognitionManagerDelegate::CheckRecognitionIsAllowed(
488 int session_id,
489 base::Callback<void(bool ask_user, bool is_allowed)> callback) {
490 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
492 const content::SpeechRecognitionSessionContext& context =
493 SpeechRecognitionManager::GetInstance()->GetSessionContext(session_id);
495 // Make sure that initiators (extensions/web pages) properly set the
496 // |render_process_id| field, which is needed later to retrieve the
497 // ChromeSpeechRecognitionPreferences associated to their profile.
498 DCHECK_NE(context.render_process_id, 0);
500 // Check that the render view type is appropriate, and whether or not we
501 // need to request permission from the user.
502 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE,
503 base::Bind(&CheckRenderViewType,
504 callback,
505 context.render_process_id,
506 context.render_view_id,
507 RequiresTrayIcon(session_id)));
510 content::SpeechRecognitionEventListener*
511 ChromeSpeechRecognitionManagerDelegate::GetEventListener() {
512 return this;
515 void ChromeSpeechRecognitionManagerDelegate::CheckRenderViewType(
516 base::Callback<void(bool ask_user, bool is_allowed)> callback,
517 int render_process_id,
518 int render_view_id,
519 bool js_api) {
520 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
521 const content::RenderViewHost* render_view_host =
522 content::RenderViewHost::FromID(render_process_id, render_view_id);
524 bool allowed = false;
525 bool ask_permission = false;
527 if (!render_view_host) {
528 if (!js_api) {
529 // If there is no render view, we cannot show the speech bubble, so this
530 // is not allowed.
531 allowed = false;
532 ask_permission = false;
533 } else {
534 // This happens for extensions. Manifest should be checked for permission.
535 allowed = true;
536 ask_permission = false;
538 BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
539 base::Bind(callback, ask_permission, allowed));
540 return;
543 WebContents* web_contents = WebContents::FromRenderViewHost(render_view_host);
544 extensions::ViewType view_type = extensions::GetViewType(web_contents);
546 // TODO(kalman): Also enable speech bubble for extension popups
547 // (VIEW_TYPE_EXTENSION_POPUP) once popup-like control UI works properly in
548 // extensions: http://crbug.com/163851.
549 // Right now the extension popup closes and dismisses immediately on user
550 // click.
551 if (view_type == extensions::VIEW_TYPE_TAB_CONTENTS ||
552 view_type == extensions::VIEW_TYPE_APP_SHELL) {
553 // If it is a tab, we can show the speech input bubble or ask for
554 // permission.
556 allowed = true;
557 if (js_api)
558 ask_permission = true;
561 BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
562 base::Bind(callback, ask_permission, allowed));
565 SpeechRecognitionBubbleController*
566 ChromeSpeechRecognitionManagerDelegate::GetBubbleController() {
567 if (!bubble_controller_.get())
568 bubble_controller_ = new SpeechRecognitionBubbleController(this);
569 return bubble_controller_.get();
572 } // namespace speech