1 // Copyright 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/search/search.h"
7 #include "base/command_line.h"
8 #include "base/metrics/field_trial.h"
9 #include "base/metrics/histogram.h"
10 #include "base/prefs/pref_service.h"
11 #include "base/strings/string_number_conversions.h"
12 #include "base/strings/string_split.h"
13 #include "base/strings/utf_string_conversions.h"
14 #include "chrome/browser/browser_process.h"
15 #include "chrome/browser/profiles/profile.h"
16 #include "chrome/browser/profiles/profile_manager.h"
17 #include "chrome/browser/search/instant_service.h"
18 #include "chrome/browser/search/instant_service_factory.h"
19 #include "chrome/browser/search_engines/template_url_service_factory.h"
20 #include "chrome/browser/search_engines/ui_thread_search_terms_data.h"
21 #include "chrome/browser/ui/browser.h"
22 #include "chrome/browser/ui/browser_instant_controller.h"
23 #include "chrome/browser/ui/browser_iterator.h"
24 #include "chrome/browser/ui/search/instant_search_prerenderer.h"
25 #include "chrome/common/chrome_switches.h"
26 #include "chrome/common/pref_names.h"
27 #include "chrome/common/search_urls.h"
28 #include "chrome/common/url_constants.h"
29 #include "components/google/core/browser/google_util.h"
30 #include "components/pref_registry/pref_registry_syncable.h"
31 #include "components/search/search.h"
32 #include "components/search_engines/template_url_service.h"
33 #include "components/sessions/serialized_navigation_entry.h"
34 #include "content/public/browser/navigation_entry.h"
35 #include "content/public/browser/render_process_host.h"
36 #include "content/public/browser/web_contents.h"
38 #if defined(ENABLE_SUPERVISED_USERS)
39 #include "chrome/browser/supervised_user/supervised_user_service.h"
40 #include "chrome/browser/supervised_user/supervised_user_service_factory.h"
41 #include "chrome/browser/supervised_user/supervised_user_url_filter.h"
48 const char kPrefetchSearchResultsOnSRP
[] = "prefetch_results_srp";
49 const char kPrerenderInstantUrlOnOmniboxFocus
[] =
50 "prerender_instant_url_on_omnibox_focus";
52 // Controls whether to use the alternate Instant search base URL. This allows
53 // experimentation of Instant search.
54 const char kUseAltInstantURL
[] = "use_alternate_instant_url";
55 const char kUseSearchPathForInstant
[] = "use_search_path_for_instant";
56 const char kAltInstantURLPath
[] = "search";
57 const char kAltInstantURLQueryParams
[] = "&qbp=1";
59 const char kShouldShowGoogleLocalNTPFlagName
[] = "google_local_ntp";
61 // Status of the New Tab URL for the default Search provider. NOTE: Used in a
62 // UMA histogram so values should only be added at the end and not reordered.
64 // Valid URL that should be used.
65 NEW_TAB_URL_VALID
= 0,
67 // Corrupt state (e.g. no profile or template url).
70 // URL should not be used because in incognito window.
71 NEW_TAB_URL_INCOGNITO
= 2,
73 // No New Tab URL set for provider.
74 NEW_TAB_URL_NOT_SET
= 3,
77 NEW_TAB_URL_INSECURE
= 4,
79 // URL should not be used because Suggest is disabled.
80 // Not used anymore, see crbug.com/340424.
81 // NEW_TAB_URL_SUGGEST_OFF = 5,
83 // URL should not be used because it is blocked for a supervised user.
84 NEW_TAB_URL_BLOCKED
= 6,
89 // Used to set the Instant support state of the Navigation entry.
90 const char kInstantSupportStateKey
[] = "instant_support_state";
92 const char kInstantSupportEnabled
[] = "Instant support enabled";
93 const char kInstantSupportDisabled
[] = "Instant support disabled";
94 const char kInstantSupportUnknown
[] = "Instant support unknown";
96 InstantSupportState
StringToInstantSupportState(const base::string16
& value
) {
97 if (value
== base::ASCIIToUTF16(kInstantSupportEnabled
))
98 return INSTANT_SUPPORT_YES
;
99 else if (value
== base::ASCIIToUTF16(kInstantSupportDisabled
))
100 return INSTANT_SUPPORT_NO
;
102 return INSTANT_SUPPORT_UNKNOWN
;
105 base::string16
InstantSupportStateToString(InstantSupportState state
) {
107 case INSTANT_SUPPORT_NO
:
108 return base::ASCIIToUTF16(kInstantSupportDisabled
);
109 case INSTANT_SUPPORT_YES
:
110 return base::ASCIIToUTF16(kInstantSupportEnabled
);
111 case INSTANT_SUPPORT_UNKNOWN
:
112 return base::ASCIIToUTF16(kInstantSupportUnknown
);
114 return base::ASCIIToUTF16(kInstantSupportUnknown
);
117 TemplateURL
* GetDefaultSearchProviderTemplateURL(Profile
* profile
) {
119 TemplateURLService
* template_url_service
=
120 TemplateURLServiceFactory::GetForProfile(profile
);
121 if (template_url_service
)
122 return template_url_service
->GetDefaultSearchProvider();
127 GURL
TemplateURLRefToGURL(const TemplateURLRef
& ref
,
128 const SearchTermsData
& search_terms_data
,
129 bool append_extra_query_params
,
130 bool force_instant_results
) {
131 TemplateURLRef::SearchTermsArgs search_terms_args
=
132 TemplateURLRef::SearchTermsArgs(base::string16());
133 search_terms_args
.append_extra_query_params
= append_extra_query_params
;
134 search_terms_args
.force_instant_results
= force_instant_results
;
135 return GURL(ref
.ReplaceSearchTerms(search_terms_args
, search_terms_data
));
138 bool MatchesAnySearchURL(const GURL
& url
,
139 TemplateURL
* template_url
,
140 const SearchTermsData
& search_terms_data
) {
141 GURL search_url
= TemplateURLRefToGURL(template_url
->url_ref(),
142 search_terms_data
, false, false);
143 if (search_url
.is_valid() && MatchesOriginAndPath(url
, search_url
))
146 // "URLCount() - 1" because we already tested url_ref above.
147 for (size_t i
= 0; i
< template_url
->URLCount() - 1; ++i
) {
148 TemplateURLRef
ref(template_url
, i
);
149 search_url
= TemplateURLRefToGURL(ref
, search_terms_data
, false, false);
150 if (search_url
.is_valid() && MatchesOriginAndPath(url
, search_url
))
157 // Returns true if |url| can be used as an Instant URL for |profile|.
158 bool IsInstantURL(const GURL
& url
, Profile
* profile
) {
159 if (!IsInstantExtendedAPIEnabled())
165 const GURL
new_tab_url(GetNewTabPageURL(profile
));
166 if (new_tab_url
.is_valid() && MatchesOriginAndPath(url
, new_tab_url
))
169 TemplateURL
* template_url
= GetDefaultSearchProviderTemplateURL(profile
);
173 if (!IsSuitableURLForInstant(url
, template_url
))
176 const TemplateURLRef
& instant_url_ref
= template_url
->instant_url_ref();
177 UIThreadSearchTermsData
search_terms_data(profile
);
178 const GURL instant_url
= TemplateURLRefToGURL(
179 instant_url_ref
, search_terms_data
, false, false);
180 if (!instant_url
.is_valid())
183 if (MatchesOriginAndPath(url
, instant_url
))
186 return IsQueryExtractionEnabled() &&
187 MatchesAnySearchURL(url
, template_url
, search_terms_data
);
190 base::string16
GetSearchTermsImpl(const content::WebContents
* contents
,
191 const content::NavigationEntry
* entry
) {
192 if (!contents
|| !IsQueryExtractionEnabled())
193 return base::string16();
195 // For security reasons, don't extract search terms if the page is not being
196 // rendered in the privileged Instant renderer process. This is to protect
197 // against a malicious page somehow scripting the search results page and
198 // faking search terms in the URL. Random pages can't get into the Instant
199 // renderer and scripting doesn't work cross-process, so if the page is in
200 // the Instant process, we know it isn't being exploited.
201 Profile
* profile
= Profile::FromBrowserContext(contents
->GetBrowserContext());
202 if (IsInstantExtendedAPIEnabled() &&
203 !IsRenderedInInstantProcess(contents
, profile
) &&
204 ((entry
== contents
->GetController().GetLastCommittedEntry()) ||
205 !ShouldAssignURLToInstantRenderer(entry
->GetURL(), profile
)))
206 return base::string16();
208 // Check to see if search terms have already been extracted.
209 base::string16 search_terms
= GetSearchTermsFromNavigationEntry(entry
);
210 if (!search_terms
.empty())
213 if (!IsQueryExtractionAllowedForURL(profile
, entry
->GetVirtualURL()))
214 return base::string16();
216 // Otherwise, extract from the URL.
217 return ExtractSearchTermsFromURL(profile
, entry
->GetVirtualURL());
220 bool IsURLAllowedForSupervisedUser(const GURL
& url
, Profile
* profile
) {
221 #if defined(ENABLE_SUPERVISED_USERS)
222 SupervisedUserService
* supervised_user_service
=
223 SupervisedUserServiceFactory::GetForProfile(profile
);
224 SupervisedUserURLFilter
* url_filter
=
225 supervised_user_service
->GetURLFilterForUIThread();
226 if (url_filter
->GetFilteringBehaviorForURL(url
) ==
227 SupervisedUserURLFilter::BLOCK
) {
234 // Returns whether |new_tab_url| can be used as a URL for the New Tab page.
235 // NEW_TAB_URL_VALID means a valid URL; other enum values imply an invalid URL.
236 NewTabURLState
IsValidNewTabURL(Profile
* profile
, const GURL
& new_tab_url
) {
237 if (profile
->IsOffTheRecord())
238 return NEW_TAB_URL_INCOGNITO
;
239 if (!new_tab_url
.is_valid())
240 return NEW_TAB_URL_NOT_SET
;
241 if (!new_tab_url
.SchemeIsCryptographic())
242 return NEW_TAB_URL_INSECURE
;
243 if (!IsURLAllowedForSupervisedUser(new_tab_url
, profile
))
244 return NEW_TAB_URL_BLOCKED
;
245 return NEW_TAB_URL_VALID
;
248 // Used to look up the URL to use for the New Tab page. Also tracks how we
249 // arrived at that URL so it can be logged with UMA.
250 struct NewTabURLDetails
{
251 NewTabURLDetails(const GURL
& url
, NewTabURLState state
)
252 : url(url
), state(state
) {}
254 static NewTabURLDetails
ForProfile(Profile
* profile
) {
255 const GURL
local_url(chrome::kChromeSearchLocalNtpUrl
);
257 base::CommandLine
* command_line
= base::CommandLine::ForCurrentProcess();
258 if (command_line
->HasSwitch(switches::kForceLocalNtp
))
259 return NewTabURLDetails(local_url
, NEW_TAB_URL_VALID
);
261 TemplateURL
* template_url
= GetDefaultSearchProviderTemplateURL(profile
);
262 if (!profile
|| !template_url
)
263 return NewTabURLDetails(local_url
, NEW_TAB_URL_BAD
);
265 GURL search_provider_url
= TemplateURLRefToGURL(
266 template_url
->new_tab_url_ref(), UIThreadSearchTermsData(profile
),
268 NewTabURLState state
= IsValidNewTabURL(profile
, search_provider_url
);
270 case NEW_TAB_URL_VALID
:
271 // We can use the search provider's page.
272 return NewTabURLDetails(search_provider_url
, state
);
273 case NEW_TAB_URL_INCOGNITO
:
274 // Incognito has its own New Tab.
275 return NewTabURLDetails(GURL(), state
);
277 // Use the local New Tab otherwise.
278 return NewTabURLDetails(local_url
, state
);
283 NewTabURLState state
;
288 base::string16
ExtractSearchTermsFromURL(Profile
* profile
, const GURL
& url
) {
289 if (url
.is_valid() && url
== GetSearchResultPrefetchBaseURL(profile
)) {
290 // InstantSearchPrerenderer has the search query for the Instant search base
292 InstantSearchPrerenderer
* prerenderer
=
293 InstantSearchPrerenderer::GetForProfile(profile
);
294 // TODO(kmadhusu): Remove this CHECK after the investigation of
297 return prerenderer
->get_last_query();
300 TemplateURL
* template_url
= GetDefaultSearchProviderTemplateURL(profile
);
301 base::string16 search_terms
;
303 template_url
->ExtractSearchTermsFromURL(
304 url
, UIThreadSearchTermsData(profile
), &search_terms
);
308 bool IsQueryExtractionAllowedForURL(Profile
* profile
, const GURL
& url
) {
309 TemplateURL
* template_url
= GetDefaultSearchProviderTemplateURL(profile
);
310 return template_url
&& IsSuitableURLForInstant(url
, template_url
);
313 base::string16
GetSearchTermsFromNavigationEntry(
314 const content::NavigationEntry
* entry
) {
315 base::string16 search_terms
;
317 entry
->GetExtraData(sessions::kSearchTermsKey
, &search_terms
);
321 base::string16
GetSearchTerms(const content::WebContents
* contents
) {
323 return base::string16();
325 const content::NavigationEntry
* entry
=
326 contents
->GetController().GetVisibleEntry();
328 return base::string16();
330 if (IsInstantExtendedAPIEnabled()) {
331 InstantSupportState state
=
332 GetInstantSupportStateFromNavigationEntry(*entry
);
333 if (state
== INSTANT_SUPPORT_NO
)
334 return base::string16();
337 return GetSearchTermsImpl(contents
, entry
);
340 bool ShouldAssignURLToInstantRenderer(const GURL
& url
, Profile
* profile
) {
341 return url
.is_valid() &&
343 IsInstantExtendedAPIEnabled() &&
344 (url
.SchemeIs(chrome::kChromeSearchScheme
) ||
345 IsInstantURL(url
, profile
));
348 bool IsRenderedInInstantProcess(const content::WebContents
* contents
,
350 const content::RenderProcessHost
* process_host
=
351 contents
->GetRenderProcessHost();
355 const InstantService
* instant_service
=
356 InstantServiceFactory::GetForProfile(profile
);
357 if (!instant_service
)
360 return instant_service
->IsInstantProcess(process_host
->GetID());
363 bool ShouldUseProcessPerSiteForInstantURL(const GURL
& url
, Profile
* profile
) {
364 return ShouldAssignURLToInstantRenderer(url
, profile
) &&
365 (url
.host() == chrome::kChromeSearchLocalNtpHost
||
366 url
.host() == chrome::kChromeSearchRemoteNtpHost
);
369 bool IsNTPURL(const GURL
& url
, Profile
* profile
) {
373 if (!IsInstantExtendedAPIEnabled())
374 return url
== GURL(chrome::kChromeUINewTabURL
);
376 const base::string16 search_terms
= ExtractSearchTermsFromURL(profile
, url
);
378 ((IsInstantURL(url
, profile
) && search_terms
.empty()) ||
379 url
== GURL(chrome::kChromeSearchLocalNtpUrl
));
382 bool IsInstantNTP(const content::WebContents
* contents
) {
386 return NavEntryIsInstantNTP(contents
,
387 contents
->GetController().GetVisibleEntry());
390 bool NavEntryIsInstantNTP(const content::WebContents
* contents
,
391 const content::NavigationEntry
* entry
) {
392 if (!contents
|| !entry
|| !IsInstantExtendedAPIEnabled())
395 Profile
* profile
= Profile::FromBrowserContext(contents
->GetBrowserContext());
396 if (!IsRenderedInInstantProcess(contents
, profile
))
399 if (entry
->GetURL() == GetLocalInstantURL(profile
))
402 GURL
new_tab_url(GetNewTabPageURL(profile
));
403 return new_tab_url
.is_valid() &&
404 MatchesOriginAndPath(entry
->GetURL(), new_tab_url
);
407 bool IsSuggestPrefEnabled(Profile
* profile
) {
408 return profile
&& !profile
->IsOffTheRecord() && profile
->GetPrefs() &&
409 profile
->GetPrefs()->GetBoolean(prefs::kSearchSuggestEnabled
);
412 GURL
GetInstantURL(Profile
* profile
, bool force_instant_results
) {
413 if (!IsInstantExtendedAPIEnabled() || !IsSuggestPrefEnabled(profile
))
416 TemplateURL
* template_url
= GetDefaultSearchProviderTemplateURL(profile
);
420 GURL instant_url
= TemplateURLRefToGURL(
421 template_url
->instant_url_ref(), UIThreadSearchTermsData(profile
),
422 true, force_instant_results
);
423 if (!instant_url
.is_valid() ||
424 !template_url
->HasSearchTermsReplacementKey(instant_url
))
427 // Extended mode requires HTTPS. Force it unless the base URL was overridden
428 // on the command line, in which case we allow HTTP (see comments on
429 // IsSuitableURLForInstant()).
430 if (!instant_url
.SchemeIsCryptographic() &&
431 !google_util::StartsWithCommandLineGoogleBaseURL(instant_url
)) {
432 GURL::Replacements replacements
;
433 replacements
.SetSchemeStr(url::kHttpsScheme
);
434 instant_url
= instant_url
.ReplaceComponents(replacements
);
437 if (!IsURLAllowedForSupervisedUser(instant_url
, profile
))
440 if (ShouldUseAltInstantURL()) {
441 GURL::Replacements replacements
;
442 const std::string
path(
443 ShouldUseSearchPathForInstant() ? kAltInstantURLPath
: std::string());
445 replacements
.SetPathStr(path
);
446 const std::string
query(
447 instant_url
.query() + std::string(kAltInstantURLQueryParams
));
448 replacements
.SetQueryStr(query
);
449 instant_url
= instant_url
.ReplaceComponents(replacements
);
454 // Returns URLs associated with the default search engine for |profile|.
455 std::vector
<GURL
> GetSearchURLs(Profile
* profile
) {
456 std::vector
<GURL
> result
;
457 TemplateURL
* template_url
= GetDefaultSearchProviderTemplateURL(profile
);
460 for (size_t i
= 0; i
< template_url
->URLCount(); ++i
) {
461 TemplateURLRef
ref(template_url
, i
);
462 result
.push_back(TemplateURLRefToGURL(ref
, UIThreadSearchTermsData(profile
),
468 GURL
GetNewTabPageURL(Profile
* profile
) {
469 return NewTabURLDetails::ForProfile(profile
).url
;
472 GURL
GetSearchResultPrefetchBaseURL(Profile
* profile
) {
473 return ShouldPrefetchSearchResults() ? GetInstantURL(profile
, true) : GURL();
476 bool ShouldPrerenderInstantUrlOnOmniboxFocus() {
477 if (!ShouldPrefetchSearchResults())
480 FieldTrialFlags flags
;
481 return GetFieldTrialInfo(&flags
) && GetBoolValueForFlagWithDefault(
482 kPrerenderInstantUrlOnOmniboxFocus
, false, flags
);
485 GURL
GetLocalInstantURL(Profile
* profile
) {
486 return GURL(chrome::kChromeSearchLocalNtpUrl
);
489 bool ShouldShowGoogleLocalNTP() {
490 FieldTrialFlags flags
;
491 return !GetFieldTrialInfo(&flags
) || GetBoolValueForFlagWithDefault(
492 kShouldShowGoogleLocalNTPFlagName
, true, flags
);
495 GURL
GetEffectiveURLForInstant(const GURL
& url
, Profile
* profile
) {
496 CHECK(ShouldAssignURLToInstantRenderer(url
, profile
))
497 << "Error granting Instant access.";
499 if (url
.SchemeIs(chrome::kChromeSearchScheme
))
502 GURL
effective_url(url
);
504 // Replace the scheme with "chrome-search:".
505 url::Replacements
<char> replacements
;
506 std::string
search_scheme(chrome::kChromeSearchScheme
);
507 replacements
.SetScheme(search_scheme
.data(),
508 url::Component(0, search_scheme
.length()));
510 // If this is the URL for a server-provided NTP, replace the host with
512 std::string
remote_ntp_host(chrome::kChromeSearchRemoteNtpHost
);
513 NewTabURLDetails details
= NewTabURLDetails::ForProfile(profile
);
514 if (details
.state
== NEW_TAB_URL_VALID
&&
515 MatchesOriginAndPath(url
, details
.url
)) {
516 replacements
.SetHost(remote_ntp_host
.c_str(),
517 url::Component(0, remote_ntp_host
.length()));
520 effective_url
= effective_url
.ReplaceComponents(replacements
);
521 return effective_url
;
524 bool HandleNewTabURLRewrite(GURL
* url
,
525 content::BrowserContext
* browser_context
) {
526 if (!IsInstantExtendedAPIEnabled())
529 if (!url
->SchemeIs(content::kChromeUIScheme
) ||
530 url
->host() != chrome::kChromeUINewTabHost
)
533 Profile
* profile
= Profile::FromBrowserContext(browser_context
);
534 NewTabURLDetails
details(NewTabURLDetails::ForProfile(profile
));
535 UMA_HISTOGRAM_ENUMERATION("NewTabPage.URLState",
536 details
.state
, NEW_TAB_URL_MAX
);
537 if (details
.url
.is_valid()) {
544 bool HandleNewTabURLReverseRewrite(GURL
* url
,
545 content::BrowserContext
* browser_context
) {
546 if (!IsInstantExtendedAPIEnabled())
549 // Do nothing in incognito.
550 Profile
* profile
= Profile::FromBrowserContext(browser_context
);
551 if (profile
&& profile
->IsOffTheRecord())
554 if (MatchesOriginAndPath(GURL(chrome::kChromeSearchLocalNtpUrl
), *url
)) {
555 *url
= GURL(chrome::kChromeUINewTabURL
);
559 GURL
new_tab_url(GetNewTabPageURL(profile
));
560 if (new_tab_url
.is_valid() && MatchesOriginAndPath(new_tab_url
, *url
)) {
561 *url
= GURL(chrome::kChromeUINewTabURL
);
568 void SetInstantSupportStateInNavigationEntry(InstantSupportState state
,
569 content::NavigationEntry
* entry
) {
573 entry
->SetExtraData(kInstantSupportStateKey
,
574 InstantSupportStateToString(state
));
577 InstantSupportState
GetInstantSupportStateFromNavigationEntry(
578 const content::NavigationEntry
& entry
) {
579 base::string16 value
;
580 if (!entry
.GetExtraData(kInstantSupportStateKey
, &value
))
581 return INSTANT_SUPPORT_UNKNOWN
;
583 return StringToInstantSupportState(value
);
586 bool ShouldPrefetchSearchResultsOnSRP() {
587 FieldTrialFlags flags
;
588 return GetFieldTrialInfo(&flags
) && GetBoolValueForFlagWithDefault(
589 kPrefetchSearchResultsOnSRP
, false, flags
);
592 bool ShouldUseAltInstantURL() {
593 FieldTrialFlags flags
;
594 return GetFieldTrialInfo(&flags
) && GetBoolValueForFlagWithDefault(
595 kUseAltInstantURL
, false, flags
);
598 bool ShouldUseSearchPathForInstant() {
599 FieldTrialFlags flags
;
600 return GetFieldTrialInfo(&flags
) && GetBoolValueForFlagWithDefault(
601 kUseSearchPathForInstant
, false, flags
);
604 } // namespace search