components/search_engines/template_url_prepopulate_data_unittest.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/command_line.h"
   6 #include "base/files/scoped_temp_dir.h"
   7 #include "base/memory/scoped_vector.h"
   8 #include "base/strings/utf_string_conversions.h"
   9 #include "components/google/core/browser/google_switches.h"
  10 #include "components/pref_registry/testing_pref_service_syncable.h"
  11 #include "components/search_engines/prepopulated_engines.h"
  12 #include "components/search_engines/search_engines_pref_names.h"
  13 #include "components/search_engines/search_terms_data.h"
  14 #include "components/search_engines/template_url.h"
  15 #include "components/search_engines/template_url_prepopulate_data.h"
  16 #include "components/search_engines/template_url_service.h"
  17 #include "testing/gtest/include/gtest/gtest.h"
  18
  19 using base::ASCIIToUTF16;
  20
  21 namespace {
  22
  23 SearchEngineType GetEngineType(const std::string& url) {
  24   TemplateURLData data;
  25   data.SetURL(url);
  26   return TemplateURLPrepopulateData::GetEngineType(TemplateURL(data),
  27                                                    SearchTermsData());
  28 }
  29
  30 std::string GetHostFromTemplateURLData(const TemplateURLData& data) {
  31   return TemplateURL(data).url_ref().GetHost(SearchTermsData());
  32 }
  33
  34 }  // namespace
  35
  36 class TemplateURLPrepopulateDataTest : public testing::Test {
  37  public:
  38   void SetUp() override {
  39     TemplateURLPrepopulateData::RegisterProfilePrefs(prefs_.registry());
  40   }
  41
  42  protected:
  43   user_prefs::TestingPrefServiceSyncable prefs_;
  44 };
  45
  46 // Verifies the set of prepopulate data doesn't contain entries with duplicate
  47 // ids.
  48 TEST_F(TemplateURLPrepopulateDataTest, UniqueIDs) {
  49   const int kCountryIds[] = {
  50       'A'<<8|'D', 'A'<<8|'E', 'A'<<8|'F', 'A'<<8|'G', 'A'<<8|'I',
  51       'A'<<8|'L', 'A'<<8|'M', 'A'<<8|'N', 'A'<<8|'O', 'A'<<8|'Q',
  52       'A'<<8|'R', 'A'<<8|'S', 'A'<<8|'T', 'A'<<8|'U', 'A'<<8|'W',
  53       'A'<<8|'X', 'A'<<8|'Z', 'B'<<8|'A', 'B'<<8|'B', 'B'<<8|'D',
  54       'B'<<8|'E', 'B'<<8|'F', 'B'<<8|'G', 'B'<<8|'H', 'B'<<8|'I',
  55       'B'<<8|'J', 'B'<<8|'M', 'B'<<8|'N', 'B'<<8|'O', 'B'<<8|'R',
  56       'B'<<8|'S', 'B'<<8|'T', 'B'<<8|'V', 'B'<<8|'W', 'B'<<8|'Y',
  57       'B'<<8|'Z', 'C'<<8|'A', 'C'<<8|'C', 'C'<<8|'D', 'C'<<8|'F',
  58       'C'<<8|'G', 'C'<<8|'H', 'C'<<8|'I', 'C'<<8|'K', 'C'<<8|'L',
  59       'C'<<8|'M', 'C'<<8|'N', 'C'<<8|'O', 'C'<<8|'R', 'C'<<8|'U',
  60       'C'<<8|'V', 'C'<<8|'X', 'C'<<8|'Y', 'C'<<8|'Z', 'D'<<8|'E',
  61       'D'<<8|'J', 'D'<<8|'K', 'D'<<8|'M', 'D'<<8|'O', 'D'<<8|'Z',
  62       'E'<<8|'C', 'E'<<8|'E', 'E'<<8|'G', 'E'<<8|'R', 'E'<<8|'S',
  63       'E'<<8|'T', 'F'<<8|'I', 'F'<<8|'J', 'F'<<8|'K', 'F'<<8|'M',
  64       'F'<<8|'O', 'F'<<8|'R', 'G'<<8|'A', 'G'<<8|'B', 'G'<<8|'D',
  65       'G'<<8|'E', 'G'<<8|'F', 'G'<<8|'G', 'G'<<8|'H', 'G'<<8|'I',
  66       'G'<<8|'L', 'G'<<8|'M', 'G'<<8|'N', 'G'<<8|'P', 'G'<<8|'Q',
  67       'G'<<8|'R', 'G'<<8|'S', 'G'<<8|'T', 'G'<<8|'U', 'G'<<8|'W',
  68       'G'<<8|'Y', 'H'<<8|'K', 'H'<<8|'M', 'H'<<8|'N', 'H'<<8|'R',
  69       'H'<<8|'T', 'H'<<8|'U', 'I'<<8|'D', 'I'<<8|'E', 'I'<<8|'L',
  70       'I'<<8|'M', 'I'<<8|'N', 'I'<<8|'O', 'I'<<8|'P', 'I'<<8|'Q',
  71       'I'<<8|'R', 'I'<<8|'S', 'I'<<8|'T', 'J'<<8|'E', 'J'<<8|'M',
  72       'J'<<8|'O', 'J'<<8|'P', 'K'<<8|'E', 'K'<<8|'G', 'K'<<8|'H',
  73       'K'<<8|'I', 'K'<<8|'M', 'K'<<8|'N', 'K'<<8|'P', 'K'<<8|'R',
  74       'K'<<8|'W', 'K'<<8|'Y', 'K'<<8|'Z', 'L'<<8|'A', 'L'<<8|'B',
  75       'L'<<8|'C', 'L'<<8|'I', 'L'<<8|'K', 'L'<<8|'R', 'L'<<8|'S',
  76       'L'<<8|'T', 'L'<<8|'U', 'L'<<8|'V', 'L'<<8|'Y', 'M'<<8|'A',
  77       'M'<<8|'C', 'M'<<8|'D', 'M'<<8|'E', 'M'<<8|'G', 'M'<<8|'H',
  78       'M'<<8|'K', 'M'<<8|'L', 'M'<<8|'M', 'M'<<8|'N', 'M'<<8|'O',
  79       'M'<<8|'P', 'M'<<8|'Q', 'M'<<8|'R', 'M'<<8|'S', 'M'<<8|'T',
  80       'M'<<8|'U', 'M'<<8|'V', 'M'<<8|'W', 'M'<<8|'X', 'M'<<8|'Y',
  81       'M'<<8|'Z', 'N'<<8|'A', 'N'<<8|'C', 'N'<<8|'E', 'N'<<8|'F',
  82       'N'<<8|'G', 'N'<<8|'I', 'N'<<8|'L', 'N'<<8|'O', 'N'<<8|'P',
  83       'N'<<8|'R', 'N'<<8|'U', 'N'<<8|'Z', 'O'<<8|'M', 'P'<<8|'A',
  84       'P'<<8|'E', 'P'<<8|'F', 'P'<<8|'G', 'P'<<8|'H', 'P'<<8|'K',
  85       'P'<<8|'L', 'P'<<8|'M', 'P'<<8|'N', 'P'<<8|'R', 'P'<<8|'S',
  86       'P'<<8|'T', 'P'<<8|'W', 'P'<<8|'Y', 'Q'<<8|'A', 'R'<<8|'E',
  87       'R'<<8|'O', 'R'<<8|'S', 'R'<<8|'U', 'R'<<8|'W', 'S'<<8|'A',
  88       'S'<<8|'B', 'S'<<8|'C', 'S'<<8|'D', 'S'<<8|'E', 'S'<<8|'G',
  89       'S'<<8|'H', 'S'<<8|'I', 'S'<<8|'J', 'S'<<8|'K', 'S'<<8|'L',
  90       'S'<<8|'M', 'S'<<8|'N', 'S'<<8|'O', 'S'<<8|'R', 'S'<<8|'T',
  91       'S'<<8|'V', 'S'<<8|'Y', 'S'<<8|'Z', 'T'<<8|'C', 'T'<<8|'D',
  92       'T'<<8|'F', 'T'<<8|'G', 'T'<<8|'H', 'T'<<8|'J', 'T'<<8|'K',
  93       'T'<<8|'L', 'T'<<8|'M', 'T'<<8|'N', 'T'<<8|'O', 'T'<<8|'R',
  94       'T'<<8|'T', 'T'<<8|'V', 'T'<<8|'W', 'T'<<8|'Z', 'U'<<8|'A',
  95       'U'<<8|'G', 'U'<<8|'M', 'U'<<8|'S', 'U'<<8|'Y', 'U'<<8|'Z',
  96       'V'<<8|'A', 'V'<<8|'C', 'V'<<8|'E', 'V'<<8|'G', 'V'<<8|'I',
  97       'V'<<8|'N', 'V'<<8|'U', 'W'<<8|'F', 'W'<<8|'S', 'Y'<<8|'E',
  98       'Y'<<8|'T', 'Z'<<8|'A', 'Z'<<8|'M', 'Z'<<8|'W', -1 };
  99
 100   for (size_t i = 0; i < arraysize(kCountryIds); ++i) {
 101     prefs_.SetInteger(prefs::kCountryIDAtInstall, kCountryIds[i]);
 102     size_t default_index;
 103     ScopedVector<TemplateURLData> urls =
 104         TemplateURLPrepopulateData::GetPrepopulatedEngines(&prefs_,
 105                                                            &default_index);
 106     std::set<int> unique_ids;
 107     for (size_t turl_i = 0; turl_i < urls.size(); ++turl_i) {
 108       ASSERT_TRUE(unique_ids.find(urls[turl_i]->prepopulate_id) ==
 109                   unique_ids.end());
 110       unique_ids.insert(urls[turl_i]->prepopulate_id);
 111     }
 112   }
 113 }
 114
 115 // Verifies that default search providers from the preferences file
 116 // override the built-in ones.
 117 TEST_F(TemplateURLPrepopulateDataTest, ProvidersFromPrefs) {
 118   prefs_.SetUserPref(prefs::kSearchProviderOverridesVersion,
 119                      new base::FundamentalValue(1));
 120   base::ListValue* overrides = new base::ListValue;
 121   scoped_ptr<base::DictionaryValue> entry(new base::DictionaryValue);
 122   // Set only the minimal required settings for a search provider configuration.
 123   entry->SetString("name", "foo");
 124   entry->SetString("keyword", "fook");
 125   entry->SetString("search_url", "http://foo.com/s?q={searchTerms}");
 126   entry->SetString("favicon_url", "http://foi.com/favicon.ico");
 127   entry->SetString("encoding", "UTF-8");
 128   entry->SetInteger("id", 1001);
 129   overrides->Append(entry->DeepCopy());
 130   prefs_.SetUserPref(prefs::kSearchProviderOverrides, overrides);
 131
 132   int version = TemplateURLPrepopulateData::GetDataVersion(&prefs_);
 133   EXPECT_EQ(1, version);
 134
 135   size_t default_index;
 136   ScopedVector<TemplateURLData> t_urls =
 137       TemplateURLPrepopulateData::GetPrepopulatedEngines(&prefs_,
 138                                                          &default_index);
 139
 140   ASSERT_EQ(1u, t_urls.size());
 141   EXPECT_EQ(ASCIIToUTF16("foo"), t_urls[0]->short_name);
 142   EXPECT_EQ(ASCIIToUTF16("fook"), t_urls[0]->keyword());
 143   EXPECT_EQ("foo.com", GetHostFromTemplateURLData(*t_urls[0]));
 144   EXPECT_EQ("foi.com", t_urls[0]->favicon_url.host());
 145   EXPECT_EQ(1u, t_urls[0]->input_encodings.size());
 146   EXPECT_EQ(1001, t_urls[0]->prepopulate_id);
 147   EXPECT_TRUE(t_urls[0]->suggestions_url.empty());
 148   EXPECT_TRUE(t_urls[0]->instant_url.empty());
 149   EXPECT_EQ(0u, t_urls[0]->alternate_urls.size());
 150   EXPECT_TRUE(t_urls[0]->search_terms_replacement_key.empty());
 151
 152   // Test the optional settings too.
 153   entry->SetString("suggest_url", "http://foo.com/suggest?q={searchTerms}");
 154   entry->SetString("instant_url", "http://foo.com/instant?q={searchTerms}");
 155   base::ListValue* alternate_urls = new base::ListValue;
 156   alternate_urls->AppendString("http://foo.com/alternate?q={searchTerms}");
 157   entry->Set("alternate_urls", alternate_urls);
 158   entry->SetString("search_terms_replacement_key", "espv");
 159   overrides = new base::ListValue;
 160   overrides->Append(entry->DeepCopy());
 161   prefs_.SetUserPref(prefs::kSearchProviderOverrides, overrides);
 162
 163   t_urls = TemplateURLPrepopulateData::GetPrepopulatedEngines(
 164       &prefs_, &default_index);
 165   ASSERT_EQ(1u, t_urls.size());
 166   EXPECT_EQ(ASCIIToUTF16("foo"), t_urls[0]->short_name);
 167   EXPECT_EQ(ASCIIToUTF16("fook"), t_urls[0]->keyword());
 168   EXPECT_EQ("foo.com", GetHostFromTemplateURLData(*t_urls[0]));
 169   EXPECT_EQ("foi.com", t_urls[0]->favicon_url.host());
 170   EXPECT_EQ(1u, t_urls[0]->input_encodings.size());
 171   EXPECT_EQ(1001, t_urls[0]->prepopulate_id);
 172   EXPECT_EQ("http://foo.com/suggest?q={searchTerms}",
 173             t_urls[0]->suggestions_url);
 174   EXPECT_EQ("http://foo.com/instant?q={searchTerms}",
 175             t_urls[0]->instant_url);
 176   ASSERT_EQ(1u, t_urls[0]->alternate_urls.size());
 177   EXPECT_EQ("http://foo.com/alternate?q={searchTerms}",
 178             t_urls[0]->alternate_urls[0]);
 179   EXPECT_EQ("espv", t_urls[0]->search_terms_replacement_key);
 180
 181   // Test that subsequent providers are loaded even if an intermediate
 182   // provider has an incomplete configuration.
 183   overrides = new base::ListValue;
 184   overrides->Append(entry->DeepCopy());
 185   entry->SetInteger("id", 1002);
 186   entry->SetString("name", "bar");
 187   entry->SetString("keyword", "bark");
 188   entry->SetString("encoding", std::string());
 189   overrides->Append(entry->DeepCopy());
 190   entry->SetInteger("id", 1003);
 191   entry->SetString("name", "baz");
 192   entry->SetString("keyword", "bazk");
 193   entry->SetString("encoding", "UTF-8");
 194   overrides->Append(entry->DeepCopy());
 195   prefs_.SetUserPref(prefs::kSearchProviderOverrides, overrides);
 196
 197   t_urls =
 198       TemplateURLPrepopulateData::GetPrepopulatedEngines(&prefs_,
 199                                                          &default_index);
 200   EXPECT_EQ(2u, t_urls.size());
 201 }
 202
 203 TEST_F(TemplateURLPrepopulateDataTest, ClearProvidersFromPrefs) {
 204   prefs_.SetUserPref(prefs::kSearchProviderOverridesVersion,
 205                      new base::FundamentalValue(1));
 206   base::ListValue* overrides = new base::ListValue;
 207   base::DictionaryValue* entry(new base::DictionaryValue);
 208   // Set only the minimal required settings for a search provider configuration.
 209   entry->SetString("name", "foo");
 210   entry->SetString("keyword", "fook");
 211   entry->SetString("search_url", "http://foo.com/s?q={searchTerms}");
 212   entry->SetString("favicon_url", "http://foi.com/favicon.ico");
 213   entry->SetString("encoding", "UTF-8");
 214   entry->SetInteger("id", 1001);
 215   overrides->Append(entry);
 216   prefs_.SetUserPref(prefs::kSearchProviderOverrides, overrides);
 217
 218   int version = TemplateURLPrepopulateData::GetDataVersion(&prefs_);
 219   EXPECT_EQ(1, version);
 220
 221   // This call removes the above search engine.
 222   TemplateURLPrepopulateData::ClearPrepopulatedEnginesInPrefs(&prefs_);
 223
 224   version = TemplateURLPrepopulateData::GetDataVersion(&prefs_);
 225   EXPECT_EQ(TemplateURLPrepopulateData::kCurrentDataVersion, version);
 226
 227   size_t default_index;
 228   ScopedVector<TemplateURLData> t_urls =
 229       TemplateURLPrepopulateData::GetPrepopulatedEngines(&prefs_,
 230                                                          &default_index);
 231   ASSERT_FALSE(t_urls.empty());
 232   for (size_t i = 0; i < t_urls.size(); ++i) {
 233     EXPECT_NE(ASCIIToUTF16("foo"), t_urls[i]->short_name);
 234     EXPECT_NE(ASCIIToUTF16("fook"), t_urls[i]->keyword());
 235     EXPECT_NE("foi.com", t_urls[i]->favicon_url.host());
 236     EXPECT_NE("foo.com", GetHostFromTemplateURLData(*t_urls[i]));
 237     EXPECT_NE(1001, t_urls[i]->prepopulate_id);
 238   }
 239   // Ensures the default URL is Google and has the optional fields filled.
 240   EXPECT_EQ(ASCIIToUTF16("Google"), t_urls[default_index]->short_name);
 241   EXPECT_FALSE(t_urls[default_index]->suggestions_url.empty());
 242   EXPECT_FALSE(t_urls[default_index]->instant_url.empty());
 243   EXPECT_FALSE(t_urls[default_index]->image_url.empty());
 244   EXPECT_FALSE(t_urls[default_index]->new_tab_url.empty());
 245   EXPECT_FALSE(t_urls[default_index]->contextual_search_url.empty());
 246   EXPECT_FALSE(t_urls[default_index]->image_url_post_params.empty());
 247   EXPECT_EQ(SEARCH_ENGINE_GOOGLE,
 248             TemplateURLPrepopulateData::GetEngineType(
 249                 TemplateURL(*t_urls[default_index]),
 250                 SearchTermsData()));
 251 }
 252
 253 // Verifies that built-in search providers are processed correctly.
 254 TEST_F(TemplateURLPrepopulateDataTest, ProvidersFromPrepopulated) {
 255   // Use United States.
 256   prefs_.SetInteger(prefs::kCountryIDAtInstall, 'U'<<8|'S');
 257   size_t default_index;
 258   ScopedVector<TemplateURLData> t_urls =
 259       TemplateURLPrepopulateData::GetPrepopulatedEngines(&prefs_,
 260                                                          &default_index);
 261
 262   // Ensure all the URLs have the required fields populated.
 263   ASSERT_FALSE(t_urls.empty());
 264   for (size_t i = 0; i < t_urls.size(); ++i) {
 265     ASSERT_FALSE(t_urls[i]->short_name.empty());
 266     ASSERT_FALSE(t_urls[i]->keyword().empty());
 267     ASSERT_FALSE(t_urls[i]->favicon_url.host().empty());
 268     ASSERT_FALSE(GetHostFromTemplateURLData(*t_urls[i]).empty());
 269     ASSERT_FALSE(t_urls[i]->input_encodings.empty());
 270     EXPECT_GT(t_urls[i]->prepopulate_id, 0);
 271   }
 272
 273   // Ensures the default URL is Google and has the optional fields filled.
 274   EXPECT_EQ(ASCIIToUTF16("Google"), t_urls[default_index]->short_name);
 275   EXPECT_FALSE(t_urls[default_index]->suggestions_url.empty());
 276   EXPECT_FALSE(t_urls[default_index]->instant_url.empty());
 277   EXPECT_FALSE(t_urls[default_index]->image_url.empty());
 278   EXPECT_FALSE(t_urls[default_index]->new_tab_url.empty());
 279   EXPECT_FALSE(t_urls[default_index]->contextual_search_url.empty());
 280   EXPECT_FALSE(t_urls[default_index]->image_url_post_params.empty());
 281   // Expect at least 2 alternate_urls.
 282   // This caught a bug with static initialization of arrays, so leave this in.
 283   EXPECT_GT(t_urls[default_index]->alternate_urls.size(), 1u);
 284   for (size_t i = 0; i < t_urls[default_index]->alternate_urls.size(); ++i)
 285     EXPECT_FALSE(t_urls[default_index]->alternate_urls[i].empty());
 286   EXPECT_EQ(SEARCH_ENGINE_GOOGLE,
 287             TemplateURLPrepopulateData::GetEngineType(
 288                 TemplateURL(*t_urls[default_index]),
 289                 SearchTermsData()));
 290   EXPECT_FALSE(t_urls[default_index]->search_terms_replacement_key.empty());
 291 }
 292
 293 TEST_F(TemplateURLPrepopulateDataTest, GetEngineTypeBasic) {
 294   EXPECT_EQ(SEARCH_ENGINE_OTHER, GetEngineType("http://example.com/"));
 295   EXPECT_EQ(SEARCH_ENGINE_ASK, GetEngineType("http://www.ask.com/"));
 296   EXPECT_EQ(SEARCH_ENGINE_OTHER, GetEngineType("http://search.atlas.cz/"));
 297   EXPECT_EQ(SEARCH_ENGINE_GOOGLE, GetEngineType("http://www.google.com/"));
 298 }
 299
 300 TEST_F(TemplateURLPrepopulateDataTest, GetEngineTypeAdvanced) {
 301   // Google URLs in different forms.
 302   const char* kGoogleURLs[] = {
 303     // Original with google:baseURL:
 304     "{google:baseURL}search?q={searchTerms}&{google:RLZ}"
 305     "{google:originalQueryForSuggestion}{google:searchFieldtrialParameter}"
 306     "sourceid=chrome&ie={inputEncoding}",
 307     // Custom with google.com and reordered query params:
 308     "http://google.com/search?{google:RLZ}{google:originalQueryForSuggestion}"
 309     "{google:searchFieldtrialParameter}"
 310     "sourceid=chrome&ie={inputEncoding}&q={searchTerms}",
 311     // Custom with a country TLD and almost no query params:
 312     "http://www.google.ru/search?q={searchTerms}"
 313   };
 314   for (size_t i = 0; i < arraysize(kGoogleURLs); ++i) {
 315     EXPECT_EQ(SEARCH_ENGINE_GOOGLE, GetEngineType(kGoogleURLs[i]));
 316   }
 317
 318   // Non-Google URLs.
 319   const char* kYahooURLs[] = {
 320       "http://search.yahoo.com/search?"
 321       "ei={inputEncoding}&fr=crmas&p={searchTerms}",
 322       "http://search.yahoo.com/search?p={searchTerms}",
 323       // Aggressively match types by checking just TLD+1.
 324       "http://someothersite.yahoo.com/",
 325   };
 326   for (size_t i = 0; i < arraysize(kYahooURLs); ++i) {
 327     EXPECT_EQ(SEARCH_ENGINE_YAHOO, GetEngineType(kYahooURLs[i]));
 328   }
 329
 330   // URLs for engines not present in country-specific lists.
 331   EXPECT_EQ(SEARCH_ENGINE_NIGMA,
 332             GetEngineType("http://nigma.ru/?s={searchTerms}&arg1=value1"));
 333   // Also test matching against alternate URLs (and TLD+1 matching).
 334   EXPECT_EQ(SEARCH_ENGINE_SOFTONIC,
 335             GetEngineType("http://test.softonic.com.br/?{searchTerms}"));
 336
 337   // Search URL for which no prepopulated search provider exists.
 338   EXPECT_EQ(SEARCH_ENGINE_OTHER,
 339             GetEngineType("http://example.net/search?q={searchTerms}"));
 340   EXPECT_EQ(SEARCH_ENGINE_OTHER, GetEngineType("invalid:search:url"));
 341
 342   // URL that doesn't look Google-related, but matches a Google base URL
 343   // specified on the command line.
 344   const std::string foo_url("http://www.foo.com/search?q={searchTerms}");
 345   EXPECT_EQ(SEARCH_ENGINE_OTHER, GetEngineType(foo_url));
 346   base::CommandLine::ForCurrentProcess()->AppendSwitchASCII(
 347       switches::kGoogleBaseURL, "http://www.foo.com/");
 348   EXPECT_EQ(SEARCH_ENGINE_GOOGLE, GetEngineType(foo_url));
 349 }