1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/speech/chrome_speech_recognition_manager_delegate.h"
10 #include "base/bind.h"
11 #include "base/prefs/pref_service.h"
12 #include "base/strings/utf_string_conversions.h"
13 #include "base/synchronization/lock.h"
14 #include "base/threading/thread_restrictions.h"
15 #include "chrome/browser/metrics/chrome_metrics_service_accessor.h"
16 #include "chrome/browser/profiles/profile_manager.h"
17 #include "chrome/browser/tab_contents/tab_util.h"
18 #include "chrome/common/pref_names.h"
19 #include "chrome/common/url_constants.h"
20 #include "content/public/browser/browser_thread.h"
21 #include "content/public/browser/render_process_host.h"
22 #include "content/public/browser/render_view_host.h"
23 #include "content/public/browser/resource_context.h"
24 #include "content/public/browser/speech_recognition_manager.h"
25 #include "content/public/browser/speech_recognition_session_config.h"
26 #include "content/public/browser/speech_recognition_session_context.h"
27 #include "content/public/browser/web_contents.h"
28 #include "content/public/browser/web_contents_observer.h"
29 #include "content/public/common/speech_recognition_error.h"
30 #include "content/public/common/speech_recognition_result.h"
31 #include "net/url_request/url_request_context_getter.h"
34 #include "chrome/installer/util/wmi.h"
37 #if defined(ENABLE_EXTENSIONS)
38 #include "chrome/browser/extensions/extension_service.h"
39 #include "extensions/browser/view_type_utils.h"
42 using content::BrowserThread
;
43 using content::SpeechRecognitionManager
;
44 using content::WebContents
;
50 void TabClosedCallbackOnIOThread(int render_process_id
, int render_view_id
) {
51 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
53 SpeechRecognitionManager
* manager
= SpeechRecognitionManager::GetInstance();
54 // |manager| becomes NULL if a browser shutdown happens between the post of
55 // this task (from the UI thread) and this call (on the IO thread). In this
56 // case we just return.
60 manager
->AbortAllSessionsForRenderView(render_process_id
, render_view_id
);
66 // Asynchronously fetches the PC and audio hardware/driver info if
67 // the user has opted into UMA. This information is sent with speech input
68 // requests to the server for identifying and improving quality issues with
69 // specific device configurations.
70 class ChromeSpeechRecognitionManagerDelegate::OptionalRequestInfo
71 : public base::RefCountedThreadSafe
<OptionalRequestInfo
> {
73 OptionalRequestInfo() : can_report_metrics_(false) {
77 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
78 // UMA opt-in can be checked only from the UI thread, so switch to that.
79 BrowserThread::PostTask(BrowserThread::UI
, FROM_HERE
,
80 base::Bind(&OptionalRequestInfo::CheckUMAAndGetHardwareInfo
, this));
83 void CheckUMAAndGetHardwareInfo() {
84 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
85 // TODO(hans): Move this check to where hardware info gets sent
87 if (ChromeMetricsServiceAccessor::IsMetricsAndCrashReportingEnabled()) {
88 // Access potentially slow OS calls from the FILE thread.
89 BrowserThread::PostTask(BrowserThread::FILE, FROM_HERE
,
90 base::Bind(&OptionalRequestInfo::GetHardwareInfo
, this));
94 void GetHardwareInfo() {
95 DCHECK_CURRENTLY_ON(BrowserThread::FILE);
96 base::AutoLock
lock(lock_
);
97 can_report_metrics_
= true;
98 base::string16 device_model
=
99 SpeechRecognitionManager::GetInstance()->GetAudioInputDeviceModel();
101 value_
= base::UTF16ToUTF8(
102 installer::WMIComputerSystem::GetModel() + L
"|" + device_model
);
103 #else // defined(OS_WIN)
104 value_
= base::UTF16ToUTF8(device_model
);
105 #endif // defined(OS_WIN)
108 std::string
value() {
109 base::AutoLock
lock(lock_
);
113 bool can_report_metrics() {
114 base::AutoLock
lock(lock_
);
115 return can_report_metrics_
;
119 friend class base::RefCountedThreadSafe
<OptionalRequestInfo
>;
121 ~OptionalRequestInfo() {}
125 bool can_report_metrics_
;
127 DISALLOW_COPY_AND_ASSIGN(OptionalRequestInfo
);
130 // Simple utility to get notified when a WebContent (a tab or an extension's
131 // background page) is closed or crashes. The callback will always be called on
133 // There is no restriction on the constructor, however this class must be
134 // destroyed on the UI thread, due to the NotificationRegistrar dependency.
135 class ChromeSpeechRecognitionManagerDelegate::TabWatcher
136 : public base::RefCountedThreadSafe
<TabWatcher
> {
138 typedef base::Callback
<void(int render_process_id
, int render_view_id
)>
141 explicit TabWatcher(TabClosedCallback tab_closed_callback
)
142 : tab_closed_callback_(tab_closed_callback
) {
145 // Starts monitoring the WebContents corresponding to the given
146 // |render_process_id|, |render_view_id| pair, invoking |tab_closed_callback_|
147 // if closed/unloaded.
148 void Watch(int render_process_id
, int render_view_id
) {
149 if (!BrowserThread::CurrentlyOn(BrowserThread::UI
)) {
150 BrowserThread::PostTask(BrowserThread::UI
, FROM_HERE
, base::Bind(
151 &TabWatcher::Watch
, this, render_process_id
, render_view_id
));
155 WebContents
* web_contents
= tab_util::GetWebContentsByID(render_process_id
,
157 // Sessions initiated by speech input extension APIs will end up in a NULL
158 // WebContent here, but they are properly managed by the
159 // chrome::SpeechInputExtensionManager. However, sessions initiated within a
160 // extension using the (new) speech JS APIs, will be properly handled here.
161 // TODO(primiano) turn this line into a DCHECK once speech input extension
162 // API is deprecated.
166 // Avoid multiple registrations for the same |web_contents|.
167 if (FindWebContents(web_contents
) != registered_web_contents_
.end())
170 registered_web_contents_
.push_back(new WebContentsTracker(
171 web_contents
, base::Bind(&TabWatcher::OnTabClosed
,
172 // |this| outlives WebContentsTracker.
173 base::Unretained(this), web_contents
),
174 render_process_id
, render_view_id
));
177 void OnTabClosed(content::WebContents
* web_contents
) {
178 ScopedVector
<WebContentsTracker
>::iterator iter
=
179 FindWebContents(web_contents
);
180 DCHECK(iter
!= registered_web_contents_
.end());
181 int render_process_id
= (*iter
)->render_process_id();
182 int render_view_id
= (*iter
)->render_view_id();
183 registered_web_contents_
.erase(iter
);
185 tab_closed_callback_
.Run(render_process_id
, render_view_id
);
189 class WebContentsTracker
: public content::WebContentsObserver
{
191 WebContentsTracker(content::WebContents
* web_contents
,
192 const base::Closure
& finished_callback
,
193 int render_process_id
,
195 : content::WebContentsObserver(web_contents
),
196 web_contents_(web_contents
),
197 finished_callback_(finished_callback
),
198 render_process_id_(render_process_id
),
199 render_view_id_(render_view_id
) {}
201 ~WebContentsTracker() override
{}
203 int render_process_id() const { return render_process_id_
; }
204 int render_view_id() const { return render_view_id_
; }
205 const content::WebContents
* GetWebContents() const { return web_contents_
; }
208 // content::WebContentsObserver overrides.
209 void WebContentsDestroyed() override
{
211 finished_callback_
.Run();
212 // NOTE: We are deleted now.
214 void RenderViewHostChanged(content::RenderViewHost
* old_host
,
215 content::RenderViewHost
* new_host
) override
{
217 finished_callback_
.Run();
218 // NOTE: We are deleted now.
221 // Raw pointer to our WebContents.
223 // Although we are a WebContentsObserver, calling
224 // WebContents::web_contents() would return NULL once we unregister
225 // ourselves in WebContentsDestroyed() or RenderViewHostChanged(). So we
226 // store a reference to perform cleanup.
227 const content::WebContents
* const web_contents_
;
228 const base::Closure finished_callback_
;
229 const int render_process_id_
;
230 const int render_view_id_
;
233 friend class base::RefCountedThreadSafe
<TabWatcher
>;
236 // Must be destroyed on the UI thread due to |registrar_| non thread-safety.
237 // TODO(lazyboy): Do we still need this?
238 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
241 // Helper function to find the iterator in |registered_web_contents_| which
242 // contains |web_contents|.
243 ScopedVector
<WebContentsTracker
>::iterator
FindWebContents(
244 content::WebContents
* web_contents
) {
245 for (ScopedVector
<WebContentsTracker
>::iterator
i(
246 registered_web_contents_
.begin());
247 i
!= registered_web_contents_
.end(); ++i
) {
248 if ((*i
)->GetWebContents() == web_contents
)
252 return registered_web_contents_
.end();
255 // Keeps track of which WebContent(s) have been registered, in order to avoid
256 // double registrations on WebContentsObserver and to pass the correct render
257 // process id and render view id to |tab_closed_callback_| after the process
259 ScopedVector
<WebContentsTracker
> registered_web_contents_
;
261 // Callback used to notify, on the thread specified by |callback_thread_| the
262 // closure of a registered tab.
263 TabClosedCallback tab_closed_callback_
;
265 DISALLOW_COPY_AND_ASSIGN(TabWatcher
);
268 ChromeSpeechRecognitionManagerDelegate
269 ::ChromeSpeechRecognitionManagerDelegate() {
272 ChromeSpeechRecognitionManagerDelegate
273 ::~ChromeSpeechRecognitionManagerDelegate() {
276 void ChromeSpeechRecognitionManagerDelegate::TabClosedCallback(
277 int render_process_id
, int render_view_id
) {
278 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
280 // Tell the S.R. Manager (which lives on the IO thread) to abort all the
281 // sessions for the given renderer view.
282 BrowserThread::PostTask(BrowserThread::IO
, FROM_HERE
, base::Bind(
283 &TabClosedCallbackOnIOThread
, render_process_id
, render_view_id
));
286 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionStart(
288 const content::SpeechRecognitionSessionContext
& context
=
289 SpeechRecognitionManager::GetInstance()->GetSessionContext(session_id
);
291 // Register callback to auto abort session on tab closure.
292 // |tab_watcher_| is lazyly istantiated on the first call.
293 if (!tab_watcher_
.get()) {
294 tab_watcher_
= new TabWatcher(
295 base::Bind(&ChromeSpeechRecognitionManagerDelegate::TabClosedCallback
,
296 base::Unretained(this)));
298 tab_watcher_
->Watch(context
.render_process_id
, context
.render_view_id
);
301 void ChromeSpeechRecognitionManagerDelegate::OnAudioStart(int session_id
) {
304 void ChromeSpeechRecognitionManagerDelegate::OnEnvironmentEstimationComplete(
308 void ChromeSpeechRecognitionManagerDelegate::OnSoundStart(int session_id
) {
311 void ChromeSpeechRecognitionManagerDelegate::OnSoundEnd(int session_id
) {
314 void ChromeSpeechRecognitionManagerDelegate::OnAudioEnd(int session_id
) {
317 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionResults(
318 int session_id
, const content::SpeechRecognitionResults
& result
) {
321 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionError(
322 int session_id
, const content::SpeechRecognitionError
& error
) {
325 void ChromeSpeechRecognitionManagerDelegate::OnAudioLevelsChange(
326 int session_id
, float volume
, float noise_volume
) {
329 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionEnd(int session_id
) {
332 void ChromeSpeechRecognitionManagerDelegate::GetDiagnosticInformation(
333 bool* can_report_metrics
,
334 std::string
* hardware_info
) {
335 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
336 if (!optional_request_info_
.get()) {
337 optional_request_info_
= new OptionalRequestInfo();
338 // Since hardware info is optional with speech input requests, we start an
339 // asynchronous fetch here and move on with recording audio. This first
340 // speech input request would send an empty string for hardware info and
341 // subsequent requests may have the hardware info available if the fetch
342 // completed before them. This way we don't end up stalling the user with
343 // a long wait and disk seeks when they click on a UI element and start
345 optional_request_info_
->Refresh();
347 *can_report_metrics
= optional_request_info_
->can_report_metrics();
348 *hardware_info
= optional_request_info_
->value();
351 void ChromeSpeechRecognitionManagerDelegate::CheckRecognitionIsAllowed(
353 base::Callback
<void(bool ask_user
, bool is_allowed
)> callback
) {
354 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
356 const content::SpeechRecognitionSessionContext
& context
=
357 SpeechRecognitionManager::GetInstance()->GetSessionContext(session_id
);
359 // Make sure that initiators (extensions/web pages) properly set the
360 // |render_process_id| field, which is needed later to retrieve the profile.
361 DCHECK_NE(context
.render_process_id
, 0);
363 int render_process_id
= context
.render_process_id
;
364 int render_view_id
= context
.render_view_id
;
365 if (context
.embedder_render_process_id
) {
366 // If this is a request originated from a guest, we need to re-route the
367 // permission check through the embedder (app).
368 render_process_id
= context
.embedder_render_process_id
;
369 render_view_id
= context
.embedder_render_view_id
;
372 // Check that the render view type is appropriate, and whether or not we
373 // need to request permission from the user.
374 BrowserThread::PostTask(BrowserThread::UI
, FROM_HERE
,
375 base::Bind(&CheckRenderViewType
,
381 content::SpeechRecognitionEventListener
*
382 ChromeSpeechRecognitionManagerDelegate::GetEventListener() {
386 bool ChromeSpeechRecognitionManagerDelegate::FilterProfanities(
387 int render_process_id
) {
388 content::RenderProcessHost
* rph
=
389 content::RenderProcessHost::FromID(render_process_id
);
390 if (!rph
) // Guard against race conditions on RPH lifetime.
393 return Profile::FromBrowserContext(rph
->GetBrowserContext())->GetPrefs()->
394 GetBoolean(prefs::kSpeechRecognitionFilterProfanities
);
398 void ChromeSpeechRecognitionManagerDelegate::CheckRenderViewType(
399 base::Callback
<void(bool ask_user
, bool is_allowed
)> callback
,
400 int render_process_id
,
401 int render_view_id
) {
402 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
403 const content::RenderViewHost
* render_view_host
=
404 content::RenderViewHost::FromID(render_process_id
, render_view_id
);
406 bool allowed
= false;
407 bool check_permission
= false;
409 if (!render_view_host
) {
410 // This happens for extensions. Manifest should be checked for permission.
412 check_permission
= false;
413 BrowserThread::PostTask(BrowserThread::IO
, FROM_HERE
,
414 base::Bind(callback
, check_permission
, allowed
));
418 #if defined(ENABLE_EXTENSIONS)
419 WebContents
* web_contents
= WebContents::FromRenderViewHost(render_view_host
);
420 extensions::ViewType view_type
= extensions::GetViewType(web_contents
);
422 if (view_type
== extensions::VIEW_TYPE_TAB_CONTENTS
||
423 view_type
== extensions::VIEW_TYPE_APP_WINDOW
||
424 view_type
== extensions::VIEW_TYPE_LAUNCHER_PAGE
||
425 view_type
== extensions::VIEW_TYPE_VIRTUAL_KEYBOARD
||
426 view_type
== extensions::VIEW_TYPE_EXTENSION_BACKGROUND_PAGE
) {
427 // If it is a tab, we can check for permission. For apps, this means
428 // manifest would be checked for permission.
430 check_permission
= true;
433 // Otherwise this should be a regular tab contents.
435 check_permission
= true;
438 BrowserThread::PostTask(BrowserThread::IO
, FROM_HERE
,
439 base::Bind(callback
, check_permission
, allowed
));
442 } // namespace speech