chrome/browser/net/predictor_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <time.h>
   6
   7 #include <algorithm>
   8 #include <sstream>
   9 #include <string>
  10
  11 #include "base/location.h"
  12 #include "base/memory/scoped_ptr.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "base/single_thread_task_runner.h"
  15 #include "base/strings/string_number_conversions.h"
  16 #include "base/thread_task_runner_handle.h"
  17 #include "base/timer/timer.h"
  18 #include "base/values.h"
  19 #include "chrome/browser/net/predictor.h"
  20 #include "chrome/browser/net/url_info.h"
  21 #include "components/network_hints/common/network_hints_common.h"
  22 #include "content/public/test/test_browser_thread.h"
  23 #include "net/base/address_list.h"
  24 #include "net/base/load_flags.h"
  25 #include "net/base/net_errors.h"
  26 #include "net/base/winsock_init.h"
  27 #include "net/dns/mock_host_resolver.h"
  28 #include "net/http/transport_security_state.h"
  29 #include "net/proxy/proxy_config_service_fixed.h"
  30 #include "net/proxy/proxy_service.h"
  31 #include "testing/gmock/include/gmock/gmock.h"
  32 #include "testing/gtest/include/gtest/gtest.h"
  33
  34 using base::Time;
  35 using base::TimeDelta;
  36 using content::BrowserThread;
  37
  38 namespace chrome_browser_net {
  39
  40 class WaitForResolutionHelper;
  41
  42 typedef base::RepeatingTimer<WaitForResolutionHelper> HelperTimer;
  43
  44 class WaitForResolutionHelper {
  45  public:
  46   WaitForResolutionHelper(Predictor* predictor, const UrlList& hosts,
  47                           HelperTimer* timer, int checks_until_quit)
  48       : predictor_(predictor),
  49         hosts_(hosts),
  50         timer_(timer),
  51         checks_until_quit_(checks_until_quit) {
  52   }
  53
  54   void CheckIfResolutionsDone() {
  55     if (--checks_until_quit_ > 0) {
  56       for (UrlList::const_iterator i = hosts_.begin(); i != hosts_.end(); ++i)
  57         if (predictor_->GetResolutionDuration(*i) ==
  58             UrlInfo::NullDuration())
  59           return;  // We don't have resolution for that host.
  60     }
  61
  62     // When all hostnames have been resolved, or we've hit the limit,
  63     // exit the loop.
  64     timer_->Stop();
  65     base::MessageLoop::current()->Quit();
  66     delete timer_;
  67     delete this;
  68   }
  69
  70  private:
  71   Predictor* predictor_;
  72   const UrlList hosts_;
  73   HelperTimer* timer_;
  74   int checks_until_quit_;
  75 };
  76
  77 class PredictorTest : public testing::Test {
  78  public:
  79   PredictorTest()
  80       : ui_thread_(BrowserThread::UI, &loop_),
  81         io_thread_(BrowserThread::IO, &loop_),
  82         host_resolver_(new net::MockCachingHostResolver()) {
  83   }
  84
  85  protected:
  86   void SetUp() override {
  87 #if defined(OS_WIN)
  88     net::EnsureWinsockInit();
  89 #endif
  90     Predictor::set_max_parallel_resolves(
  91         Predictor::kMaxSpeculativeParallelResolves);
  92     Predictor::set_max_queueing_delay(
  93         Predictor::kMaxSpeculativeResolveQueueDelayMs);
  94     // Since we are using a caching HostResolver, the following latencies will
  95     // only be incurred by the first request, after which the result will be
  96     // cached internally by |host_resolver_|.
  97     net::RuleBasedHostResolverProc* rules = host_resolver_->rules();
  98     rules->AddRuleWithLatency("www.google.com", "127.0.0.1", 50);
  99     rules->AddRuleWithLatency("gmail.google.com.com", "127.0.0.1", 70);
 100     rules->AddRuleWithLatency("mail.google.com", "127.0.0.1", 44);
 101     rules->AddRuleWithLatency("gmail.com", "127.0.0.1", 63);
 102   }
 103
 104   void WaitForResolution(Predictor* predictor, const UrlList& hosts) {
 105     HelperTimer* timer = new HelperTimer();
 106     // By default allow the loop to run for a minute -- 600 iterations.
 107     timer->Start(FROM_HERE, TimeDelta::FromMilliseconds(100),
 108                  new WaitForResolutionHelper(predictor, hosts, timer, 600),
 109                  &WaitForResolutionHelper::CheckIfResolutionsDone);
 110     base::MessageLoop::current()->Run();
 111   }
 112
 113   void WaitForResolutionWithLimit(
 114       Predictor* predictor, const UrlList& hosts, int limit) {
 115     HelperTimer* timer = new HelperTimer();
 116     timer->Start(FROM_HERE, TimeDelta::FromMilliseconds(100),
 117                  new WaitForResolutionHelper(predictor, hosts, timer, limit),
 118                  &WaitForResolutionHelper::CheckIfResolutionsDone);
 119     base::MessageLoop::current()->Run();
 120   }
 121
 122  private:
 123   // IMPORTANT: do not move this below |host_resolver_|; the host resolver
 124   // must not outlive the message loop, otherwise bad things can happen
 125   // (like posting to a deleted message loop).
 126   base::MessageLoopForUI loop_;
 127   content::TestBrowserThread ui_thread_;
 128   content::TestBrowserThread io_thread_;
 129
 130  protected:
 131   scoped_ptr<net::MockCachingHostResolver> host_resolver_;
 132 };
 133
 134 //------------------------------------------------------------------------------
 135
 136 TEST_F(PredictorTest, StartupShutdownTest) {
 137   Predictor testing_master(true, true);
 138   testing_master.Shutdown();
 139 }
 140
 141
 142 TEST_F(PredictorTest, ShutdownWhenResolutionIsPendingTest) {
 143   scoped_ptr<net::HostResolver> host_resolver(new net::HangingHostResolver());
 144
 145   Predictor testing_master(true, true);
 146   testing_master.SetHostResolver(host_resolver.get());
 147
 148   GURL localhost("http://localhost:80");
 149   UrlList names;
 150   names.push_back(localhost);
 151
 152   testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 153
 154   base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 155       FROM_HERE, base::MessageLoop::QuitClosure(),
 156       base::TimeDelta::FromMilliseconds(500));
 157   base::MessageLoop::current()->Run();
 158
 159   EXPECT_FALSE(testing_master.WasFound(localhost));
 160
 161   testing_master.Shutdown();
 162
 163   // Clean up after ourselves.
 164   base::MessageLoop::current()->RunUntilIdle();
 165 }
 166
 167 TEST_F(PredictorTest, SingleLookupTest) {
 168   Predictor testing_master(true, true);
 169   testing_master.SetHostResolver(host_resolver_.get());
 170
 171   GURL goog("http://www.google.com:80");
 172
 173   UrlList names;
 174   names.push_back(goog);
 175
 176   // Try to flood the predictor with many concurrent requests.
 177   for (int i = 0; i < 10; i++)
 178     testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 179
 180   WaitForResolution(&testing_master, names);
 181
 182   EXPECT_TRUE(testing_master.WasFound(goog));
 183
 184   base::MessageLoop::current()->RunUntilIdle();
 185
 186   EXPECT_GT(testing_master.peak_pending_lookups(), names.size() / 2);
 187   EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
 188   EXPECT_LE(testing_master.peak_pending_lookups(),
 189             testing_master.max_concurrent_dns_lookups());
 190
 191   testing_master.Shutdown();
 192 }
 193
 194 TEST_F(PredictorTest, ConcurrentLookupTest) {
 195   host_resolver_->rules()->AddSimulatedFailure("*.notfound");
 196
 197   Predictor testing_master(true, true);
 198   testing_master.SetHostResolver(host_resolver_.get());
 199
 200   GURL goog("http://www.google.com:80"),
 201       goog2("http://gmail.google.com.com:80"),
 202       goog3("http://mail.google.com:80"),
 203       goog4("http://gmail.com:80");
 204   GURL bad1("http://bad1.notfound:80"),
 205       bad2("http://bad2.notfound:80");
 206
 207   UrlList names;
 208   names.push_back(goog);
 209   names.push_back(goog3);
 210   names.push_back(bad1);
 211   names.push_back(goog2);
 212   names.push_back(bad2);
 213   names.push_back(goog4);
 214   names.push_back(goog);
 215
 216   // Try to flood the predictor with many concurrent requests.
 217   for (int i = 0; i < 10; i++)
 218     testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 219
 220   WaitForResolution(&testing_master, names);
 221
 222   EXPECT_TRUE(testing_master.WasFound(goog));
 223   EXPECT_TRUE(testing_master.WasFound(goog3));
 224   EXPECT_TRUE(testing_master.WasFound(goog2));
 225   EXPECT_TRUE(testing_master.WasFound(goog4));
 226   EXPECT_FALSE(testing_master.WasFound(bad1));
 227   EXPECT_FALSE(testing_master.WasFound(bad2));
 228
 229   base::MessageLoop::current()->RunUntilIdle();
 230
 231   EXPECT_FALSE(testing_master.WasFound(bad1));
 232   EXPECT_FALSE(testing_master.WasFound(bad2));
 233
 234   EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
 235   EXPECT_LE(testing_master.peak_pending_lookups(),
 236             testing_master.max_concurrent_dns_lookups());
 237
 238   testing_master.Shutdown();
 239 }
 240
 241 TEST_F(PredictorTest, MassiveConcurrentLookupTest) {
 242   host_resolver_->rules()->AddSimulatedFailure("*.notfound");
 243
 244   Predictor testing_master(true, true);
 245   testing_master.SetHostResolver(host_resolver_.get());
 246
 247   UrlList names;
 248   for (int i = 0; i < 100; i++)
 249     names.push_back(GURL(
 250         "http://host" + base::IntToString(i) + ".notfound:80"));
 251
 252   // Try to flood the predictor with many concurrent requests.
 253   for (int i = 0; i < 10; i++)
 254     testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 255
 256   WaitForResolution(&testing_master, names);
 257
 258   base::MessageLoop::current()->RunUntilIdle();
 259
 260   EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
 261   EXPECT_LE(testing_master.peak_pending_lookups(),
 262             testing_master.max_concurrent_dns_lookups());
 263
 264   testing_master.Shutdown();
 265 }
 266
 267 //------------------------------------------------------------------------------
 268 // Functions to help synthesize and test serializations of subresource referrer
 269 // lists.
 270
 271 // Return a motivation_list if we can find one for the given motivating_host (or
 272 // NULL if a match is not found).
 273 static const base::ListValue* FindSerializationMotivation(
 274     const GURL& motivation,
 275     const base::ListValue* referral_list) {
 276   CHECK_LT(0u, referral_list->GetSize());  // Room for version.
 277   int format_version = -1;
 278   CHECK(referral_list->GetInteger(0, &format_version));
 279   CHECK_EQ(Predictor::kPredictorReferrerVersion, format_version);
 280   const base::ListValue* motivation_list(NULL);
 281   for (size_t i = 1; i < referral_list->GetSize(); ++i) {
 282     referral_list->GetList(i, &motivation_list);
 283     std::string existing_spec;
 284     EXPECT_TRUE(motivation_list->GetString(0, &existing_spec));
 285     if (motivation == GURL(existing_spec))
 286       return motivation_list;
 287   }
 288   return NULL;
 289 }
 290
 291 static base::ListValue* FindSerializationMotivation(
 292     const GURL& motivation,
 293     base::ListValue* referral_list) {
 294   return const_cast<base::ListValue*>(FindSerializationMotivation(
 295       motivation, static_cast<const base::ListValue*>(referral_list)));
 296 }
 297
 298 // Create a new empty serialization list.
 299 static base::ListValue* NewEmptySerializationList() {
 300   base::ListValue* list = new base::ListValue;
 301   list->Append(
 302       new base::FundamentalValue(Predictor::kPredictorReferrerVersion));
 303   return list;
 304 }
 305
 306 // Add a motivating_url and a subresource_url to a serialized list, using
 307 // this given latency. This is a helper function for quickly building these
 308 // lists.
 309 static void AddToSerializedList(const GURL& motivation,
 310                                 const GURL& subresource,
 311                                 double use_rate,
 312                                 base::ListValue* referral_list) {
 313   // Find the motivation if it is already used.
 314   base::ListValue* motivation_list = FindSerializationMotivation(motivation,
 315                                                            referral_list);
 316   if (!motivation_list) {
 317     // This is the first mention of this motivation, so build a list.
 318     motivation_list = new base::ListValue;
 319     motivation_list->Append(new base::StringValue(motivation.spec()));
 320     // Provide empty subresource list.
 321     motivation_list->Append(new base::ListValue());
 322
 323     // ...and make it part of the serialized referral_list.
 324     referral_list->Append(motivation_list);
 325   }
 326
 327   base::ListValue* subresource_list(NULL);
 328   // 0 == url; 1 == subresource_list.
 329   EXPECT_TRUE(motivation_list->GetList(1, &subresource_list));
 330
 331   // We won't bother to check for the subresource being there already.  Worst
 332   // case, during deserialization, the latency value we supply plus the
 333   // existing value(s) will be added to the referrer.
 334
 335   subresource_list->Append(new base::StringValue(subresource.spec()));
 336   subresource_list->Append(new base::FundamentalValue(use_rate));
 337 }
 338
 339 // For a given motivation, and subresource, find what latency is currently
 340 // listed.  This assume a well formed serialization, which has at most one such
 341 // entry for any pair of names.  If no such pair is found, then return false.
 342 // Data is written into use_rate arguments.
 343 static bool GetDataFromSerialization(const GURL& motivation,
 344                                      const GURL& subresource,
 345                                      const base::ListValue& referral_list,
 346                                      double* use_rate) {
 347   const base::ListValue* motivation_list =
 348       FindSerializationMotivation(motivation, &referral_list);
 349   if (!motivation_list)
 350     return false;
 351   const base::ListValue* subresource_list;
 352   EXPECT_TRUE(motivation_list->GetList(1, &subresource_list));
 353   for (size_t i = 0; i < subresource_list->GetSize();) {
 354     std::string url_spec;
 355     EXPECT_TRUE(subresource_list->GetString(i++, &url_spec));
 356     EXPECT_TRUE(subresource_list->GetDouble(i++, use_rate));
 357     if (subresource == GURL(url_spec)) {
 358       return true;
 359     }
 360   }
 361   return false;
 362 }
 363
 364 //------------------------------------------------------------------------------
 365
 366 // Make sure nil referral lists really have no entries, and no latency listed.
 367 TEST_F(PredictorTest, ReferrerSerializationNilTest) {
 368   Predictor predictor(true, true);
 369   predictor.SetHostResolver(host_resolver_.get());
 370
 371   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 372   predictor.SerializeReferrers(referral_list.get());
 373   EXPECT_EQ(1U, referral_list->GetSize());
 374   EXPECT_FALSE(GetDataFromSerialization(
 375     GURL("http://a.com:79"), GURL("http://b.com:78"),
 376       *referral_list.get(), NULL));
 377
 378   predictor.Shutdown();
 379 }
 380
 381 // Make sure that when a serialization list includes a value, that it can be
 382 // deserialized into the database, and can be extracted back out via
 383 // serialization without being changed.
 384 TEST_F(PredictorTest, ReferrerSerializationSingleReferrerTest) {
 385   Predictor predictor(true, true);
 386   predictor.SetHostResolver(host_resolver_.get());
 387   const GURL motivation_url("http://www.google.com:91");
 388   const GURL subresource_url("http://icons.google.com:90");
 389   const double kUseRate = 23.4;
 390   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 391
 392   AddToSerializedList(motivation_url, subresource_url,
 393       kUseRate, referral_list.get());
 394
 395   predictor.DeserializeReferrers(*referral_list.get());
 396
 397   base::ListValue recovered_referral_list;
 398   predictor.SerializeReferrers(&recovered_referral_list);
 399   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 400   double rate;
 401   EXPECT_TRUE(GetDataFromSerialization(
 402       motivation_url, subresource_url, recovered_referral_list, &rate));
 403   EXPECT_EQ(rate, kUseRate);
 404
 405   predictor.Shutdown();
 406 }
 407
 408 // Check that GetHtmlReferrerLists() doesn't crash when given duplicated
 409 // domains for referring URL, and that it sorts the results in the
 410 // correct order.
 411 TEST_F(PredictorTest, GetHtmlReferrerLists) {
 412   Predictor predictor(true, true);
 413   predictor.SetHostResolver(host_resolver_.get());
 414   const double kUseRate = 23.4;
 415   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 416
 417   AddToSerializedList(
 418       GURL("http://d.google.com/x1"),
 419       GURL("http://foo.com/"),
 420       kUseRate, referral_list.get());
 421
 422   // Duplicated hostname (d.google.com). This should not cause any crashes
 423   // (i.e. crbug.com/116345)
 424   AddToSerializedList(
 425       GURL("http://d.google.com/x2"),
 426       GURL("http://foo.com/"),
 427       kUseRate, referral_list.get());
 428
 429   AddToSerializedList(
 430       GURL("http://a.yahoo.com/y"),
 431       GURL("http://foo1.com/"),
 432       kUseRate, referral_list.get());
 433
 434   AddToSerializedList(
 435       GURL("http://b.google.com/x3"),
 436       GURL("http://foo2.com/"),
 437       kUseRate, referral_list.get());
 438
 439   AddToSerializedList(
 440       GURL("http://d.yahoo.com/x5"),
 441       GURL("http://i.like.turtles/"),
 442       kUseRate, referral_list.get());
 443
 444   AddToSerializedList(
 445       GURL("http://c.yahoo.com/x4"),
 446       GURL("http://foo3.com/"),
 447       kUseRate, referral_list.get());
 448
 449   predictor.DeserializeReferrers(*referral_list.get());
 450
 451   std::string html;
 452   predictor.GetHtmlReferrerLists(&html);
 453
 454   // The lexicographic sorting of hostnames would be:
 455   //   a.yahoo.com
 456   //   b.google.com
 457   //   c.yahoo.com
 458   //   d.google.com
 459   //   d.yahoo.com
 460   //
 461   // However we expect to sort them by domain in the output:
 462   //   b.google.com
 463   //   d.google.com
 464   //   a.yahoo.com
 465   //   c.yahoo.com
 466   //   d.yahoo.com
 467
 468   size_t pos[] = {
 469       html.find("<td rowspan=1>http://b.google.com/x3"),
 470       html.find("<td rowspan=1>http://d.google.com/x1"),
 471       html.find("<td rowspan=1>http://d.google.com/x2"),
 472       html.find("<td rowspan=1>http://a.yahoo.com/y"),
 473       html.find("<td rowspan=1>http://c.yahoo.com/x4"),
 474       html.find("<td rowspan=1>http://d.yahoo.com/x5"),
 475   };
 476
 477   // Make sure things appeared in the expected order.
 478   for (size_t i = 1; i < arraysize(pos); ++i) {
 479     EXPECT_LT(pos[i - 1], pos[i]) << "Mismatch for pos[" << i << "]";
 480   }
 481
 482   predictor.Shutdown();
 483 }
 484
 485 // Verify that two floats are within 1% of each other in value.
 486 #define EXPECT_SIMILAR(a, b) do { \
 487     double espilon_ratio = 1.01;  \
 488     if ((a) < 0.)  \
 489       espilon_ratio = 1 / espilon_ratio;  \
 490     EXPECT_LT(a, espilon_ratio * (b));   \
 491     EXPECT_GT((a) * espilon_ratio, b);   \
 492     } while (0)
 493
 494
 495 // Make sure the Trim() functionality works as expected.
 496 TEST_F(PredictorTest, ReferrerSerializationTrimTest) {
 497   Predictor predictor(true, true);
 498   predictor.SetHostResolver(host_resolver_.get());
 499   GURL motivation_url("http://www.google.com:110");
 500
 501   GURL icon_subresource_url("http://icons.google.com:111");
 502   const double kRateIcon = 16.0 * Predictor::kDiscardableExpectedValue;
 503   GURL img_subresource_url("http://img.google.com:118");
 504   const double kRateImg = 8.0 * Predictor::kDiscardableExpectedValue;
 505
 506   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 507   AddToSerializedList(
 508       motivation_url, icon_subresource_url, kRateIcon, referral_list.get());
 509   AddToSerializedList(
 510       motivation_url, img_subresource_url, kRateImg, referral_list.get());
 511
 512   predictor.DeserializeReferrers(*referral_list.get());
 513
 514   base::ListValue recovered_referral_list;
 515   predictor.SerializeReferrers(&recovered_referral_list);
 516   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 517   double rate;
 518   EXPECT_TRUE(GetDataFromSerialization(
 519       motivation_url, icon_subresource_url, recovered_referral_list,
 520       &rate));
 521   EXPECT_SIMILAR(rate, kRateIcon);
 522
 523   EXPECT_TRUE(GetDataFromSerialization(
 524       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 525   EXPECT_SIMILAR(rate, kRateImg);
 526
 527   // Each time we Trim 24 times, the user_rate figures should reduce by a factor
 528   // of two,  until they are small, and then a trim will delete the whole entry.
 529   for (int i = 0; i < 24; ++i)
 530     predictor.TrimReferrersNow();
 531   predictor.SerializeReferrers(&recovered_referral_list);
 532   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 533   EXPECT_TRUE(GetDataFromSerialization(
 534       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 535   EXPECT_SIMILAR(rate, kRateIcon / 2);
 536
 537   EXPECT_TRUE(GetDataFromSerialization(
 538       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 539   EXPECT_SIMILAR(rate, kRateImg / 2);
 540
 541   for (int i = 0; i < 24; ++i)
 542     predictor.TrimReferrersNow();
 543   predictor.SerializeReferrers(&recovered_referral_list);
 544   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 545   EXPECT_TRUE(GetDataFromSerialization(
 546       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 547   EXPECT_SIMILAR(rate, kRateIcon / 4);
 548   EXPECT_TRUE(GetDataFromSerialization(
 549       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 550   EXPECT_SIMILAR(rate, kRateImg / 4);
 551
 552   for (int i = 0; i < 24; ++i)
 553     predictor.TrimReferrersNow();
 554   predictor.SerializeReferrers(&recovered_referral_list);
 555   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 556   EXPECT_TRUE(GetDataFromSerialization(
 557       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 558   EXPECT_SIMILAR(rate, kRateIcon / 8);
 559
 560   // Img is below threshold, and so it gets deleted.
 561   EXPECT_FALSE(GetDataFromSerialization(
 562       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 563
 564   for (int i = 0; i < 24; ++i)
 565     predictor.TrimReferrersNow();
 566   predictor.SerializeReferrers(&recovered_referral_list);
 567   // Icon is also trimmed away, so entire set gets discarded.
 568   EXPECT_EQ(1U, recovered_referral_list.GetSize());
 569   EXPECT_FALSE(GetDataFromSerialization(
 570       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 571   EXPECT_FALSE(GetDataFromSerialization(
 572       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 573
 574   predictor.Shutdown();
 575 }
 576
 577
 578 TEST_F(PredictorTest, PriorityQueuePushPopTest) {
 579   Predictor::HostNameQueue queue;
 580
 581   GURL first("http://first:80"), second("http://second:90");
 582
 583   // First check high priority queue FIFO functionality.
 584   EXPECT_TRUE(queue.IsEmpty());
 585   queue.Push(first, UrlInfo::LEARNED_REFERAL_MOTIVATED);
 586   EXPECT_FALSE(queue.IsEmpty());
 587   queue.Push(second, UrlInfo::MOUSE_OVER_MOTIVATED);
 588   EXPECT_FALSE(queue.IsEmpty());
 589   EXPECT_EQ(queue.Pop(), first);
 590   EXPECT_FALSE(queue.IsEmpty());
 591   EXPECT_EQ(queue.Pop(), second);
 592   EXPECT_TRUE(queue.IsEmpty());
 593
 594   // Then check low priority queue FIFO functionality.
 595   queue.Push(first, UrlInfo::PAGE_SCAN_MOTIVATED);
 596   EXPECT_FALSE(queue.IsEmpty());
 597   queue.Push(second, UrlInfo::OMNIBOX_MOTIVATED);
 598   EXPECT_FALSE(queue.IsEmpty());
 599   EXPECT_EQ(queue.Pop(), first);
 600   EXPECT_FALSE(queue.IsEmpty());
 601   EXPECT_EQ(queue.Pop(), second);
 602   EXPECT_TRUE(queue.IsEmpty());
 603 }
 604
 605 TEST_F(PredictorTest, PriorityQueueReorderTest) {
 606   Predictor::HostNameQueue queue;
 607
 608   // Push all the low priority items.
 609   GURL low1("http://low1:80"),
 610       low2("http://low2:80"),
 611       low3("http://low3:443"),
 612       low4("http://low4:80"),
 613       low5("http://low5:80"),
 614       hi1("http://hi1:80"),
 615       hi2("http://hi2:80"),
 616       hi3("http://hi3:80");
 617
 618   EXPECT_TRUE(queue.IsEmpty());
 619   queue.Push(low1, UrlInfo::PAGE_SCAN_MOTIVATED);
 620   queue.Push(low2, UrlInfo::UNIT_TEST_MOTIVATED);
 621   queue.Push(low3, UrlInfo::LINKED_MAX_MOTIVATED);
 622   queue.Push(low4, UrlInfo::OMNIBOX_MOTIVATED);
 623   queue.Push(low5, UrlInfo::STARTUP_LIST_MOTIVATED);
 624   queue.Push(low4, UrlInfo::OMNIBOX_MOTIVATED);
 625
 626   // Push all the high prority items
 627   queue.Push(hi1, UrlInfo::LEARNED_REFERAL_MOTIVATED);
 628   queue.Push(hi2, UrlInfo::STATIC_REFERAL_MOTIVATED);
 629   queue.Push(hi3, UrlInfo::MOUSE_OVER_MOTIVATED);
 630
 631   // Check that high priority stuff comes out first, and in FIFO order.
 632   EXPECT_EQ(queue.Pop(), hi1);
 633   EXPECT_EQ(queue.Pop(), hi2);
 634   EXPECT_EQ(queue.Pop(), hi3);
 635
 636   // ...and then low priority strings.
 637   EXPECT_EQ(queue.Pop(), low1);
 638   EXPECT_EQ(queue.Pop(), low2);
 639   EXPECT_EQ(queue.Pop(), low3);
 640   EXPECT_EQ(queue.Pop(), low4);
 641   EXPECT_EQ(queue.Pop(), low5);
 642   EXPECT_EQ(queue.Pop(), low4);
 643
 644   EXPECT_TRUE(queue.IsEmpty());
 645 }
 646
 647 TEST_F(PredictorTest, CanonicalizeUrl) {
 648   // Base case, only handles HTTP and HTTPS.
 649   EXPECT_EQ(GURL(), Predictor::CanonicalizeUrl(GURL("ftp://anything")));
 650
 651   // Remove path testing.
 652   GURL long_url("http://host:999/path?query=value");
 653   EXPECT_EQ(Predictor::CanonicalizeUrl(long_url), long_url.GetWithEmptyPath());
 654
 655   // Default port cannoncalization.
 656   GURL implied_port("http://test");
 657   GURL explicit_port("http://test:80");
 658   EXPECT_EQ(Predictor::CanonicalizeUrl(implied_port),
 659             Predictor::CanonicalizeUrl(explicit_port));
 660
 661   // Port is still maintained.
 662   GURL port_80("http://test:80");
 663   GURL port_90("http://test:90");
 664   EXPECT_NE(Predictor::CanonicalizeUrl(port_80),
 665             Predictor::CanonicalizeUrl(port_90));
 666
 667   // Host is still maintained.
 668   GURL host_1("http://test_1");
 669   GURL host_2("http://test_2");
 670   EXPECT_NE(Predictor::CanonicalizeUrl(host_1),
 671             Predictor::CanonicalizeUrl(host_2));
 672
 673   // Scheme is maintained (mismatch identified).
 674   GURL http("http://test");
 675   GURL https("https://test");
 676   EXPECT_NE(Predictor::CanonicalizeUrl(http),
 677             Predictor::CanonicalizeUrl(https));
 678
 679   // Https works fine.
 680   GURL long_https("https://host:999/path?query=value");
 681   EXPECT_EQ(Predictor::CanonicalizeUrl(long_https),
 682             long_https.GetWithEmptyPath());
 683 }
 684
 685 TEST_F(PredictorTest, DiscardPredictorResults) {
 686   SimplePredictor predictor(true, true);
 687   predictor.SetHostResolver(host_resolver_.get());
 688   base::ListValue referral_list;
 689   predictor.SerializeReferrers(&referral_list);
 690   EXPECT_EQ(1U, referral_list.GetSize());
 691
 692   GURL host_1("http://test_1");
 693   GURL host_2("http://test_2");
 694   predictor.LearnFromNavigation(host_1, host_2);
 695
 696   predictor.SerializeReferrers(&referral_list);
 697   EXPECT_EQ(2U, referral_list.GetSize());
 698
 699   predictor.DiscardAllResults();
 700   predictor.SerializeReferrers(&referral_list);
 701   EXPECT_EQ(1U, referral_list.GetSize());
 702
 703   predictor.Shutdown();
 704 }
 705
 706 class TestPredictorObserver : public PredictorObserver {
 707  public:
 708   // PredictorObserver implementation:
 709   void OnPreconnectUrl(const GURL& url,
 710                        const GURL& first_party_for_cookies,
 711                        UrlInfo::ResolutionMotivation motivation,
 712                        int count) override {
 713     preconnected_urls_.push_back(url);
 714   }
 715
 716   std::vector<GURL> preconnected_urls_;
 717 };
 718
 719 // Tests that preconnects apply the HSTS list.
 720 TEST_F(PredictorTest, HSTSRedirect) {
 721   const GURL kHttpUrl("http://example.com");
 722   const GURL kHttpsUrl("https://example.com");
 723
 724   const base::Time expiry =
 725       base::Time::Now() + base::TimeDelta::FromSeconds(1000);
 726   net::TransportSecurityState state;
 727   state.AddHSTS(kHttpUrl.host(), expiry, false);
 728
 729   Predictor predictor(true, true);
 730   TestPredictorObserver observer;
 731   predictor.SetObserver(&observer);
 732   predictor.SetTransportSecurityState(&state);
 733
 734   predictor.PreconnectUrl(kHttpUrl, GURL(), UrlInfo::OMNIBOX_MOTIVATED, 2);
 735   ASSERT_EQ(1u, observer.preconnected_urls_.size());
 736   EXPECT_EQ(kHttpsUrl, observer.preconnected_urls_[0]);
 737
 738   predictor.Shutdown();
 739 }
 740
 741 // Tests that preconnecting a URL on the HSTS list preconnects the subresources
 742 // for the SSL version.
 743 TEST_F(PredictorTest, HSTSRedirectSubresources) {
 744   const GURL kHttpUrl("http://example.com");
 745   const GURL kHttpsUrl("https://example.com");
 746   const GURL kSubresourceUrl("https://images.example.com");
 747   const double kUseRate = 23.4;
 748
 749   const base::Time expiry =
 750       base::Time::Now() + base::TimeDelta::FromSeconds(1000);
 751   net::TransportSecurityState state;
 752   state.AddHSTS(kHttpUrl.host(), expiry, false);
 753
 754   SimplePredictor predictor(true, true);
 755   TestPredictorObserver observer;
 756   predictor.SetObserver(&observer);
 757   predictor.SetTransportSecurityState(&state);
 758
 759   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 760   AddToSerializedList(
 761       kHttpsUrl, kSubresourceUrl, kUseRate, referral_list.get());
 762   predictor.DeserializeReferrers(*referral_list.get());
 763
 764   predictor.PreconnectUrlAndSubresources(kHttpUrl, GURL());
 765   ASSERT_EQ(2u, observer.preconnected_urls_.size());
 766   EXPECT_EQ(kHttpsUrl, observer.preconnected_urls_[0]);
 767   EXPECT_EQ(kSubresourceUrl, observer.preconnected_urls_[1]);
 768
 769   predictor.Shutdown();
 770 }
 771
 772 TEST_F(PredictorTest, NoProxyService) {
 773   // Don't actually try to resolve names.
 774   Predictor::set_max_parallel_resolves(0);
 775
 776   Predictor testing_master(true, true);
 777
 778   GURL goog("http://www.google.com:80");
 779   testing_master.Resolve(goog, UrlInfo::OMNIBOX_MOTIVATED);
 780   EXPECT_FALSE(testing_master.work_queue_.IsEmpty());
 781
 782   testing_master.Shutdown();
 783 }
 784
 785 TEST_F(PredictorTest, ProxyDefinitelyEnabled) {
 786   // Don't actually try to resolve names.
 787   Predictor::set_max_parallel_resolves(0);
 788
 789   Predictor testing_master(true, true);
 790
 791   net::ProxyConfig config;
 792   config.proxy_rules().ParseFromString("http=socks://localhost:12345");
 793   testing_master.proxy_service_ = net::ProxyService::CreateFixed(config);
 794
 795   GURL goog("http://www.google.com:80");
 796   testing_master.Resolve(goog, UrlInfo::OMNIBOX_MOTIVATED);
 797
 798   // Proxy is definitely in use, so there is no need to pre-resolve the domain.
 799   EXPECT_TRUE(testing_master.work_queue_.IsEmpty());
 800
 801   delete testing_master.proxy_service_;
 802   testing_master.Shutdown();
 803 }
 804
 805 TEST_F(PredictorTest, ProxyDefinitelyNotEnabled) {
 806   // Don't actually try to resolve names.
 807   Predictor::set_max_parallel_resolves(0);
 808
 809   Predictor testing_master(true, true);
 810   net::ProxyConfig config = net::ProxyConfig::CreateDirect();
 811   testing_master.proxy_service_ = net::ProxyService::CreateFixed(config);
 812
 813   GURL goog("http://www.google.com:80");
 814   testing_master.Resolve(goog, UrlInfo::OMNIBOX_MOTIVATED);
 815
 816   // Proxy is not in use, so the name has been registered for pre-resolve.
 817   EXPECT_FALSE(testing_master.work_queue_.IsEmpty());
 818
 819   delete testing_master.proxy_service_;
 820   testing_master.Shutdown();
 821 }
 822
 823 TEST_F(PredictorTest, ProxyMaybeEnabled) {
 824   // Don't actually try to resolve names.
 825   Predictor::set_max_parallel_resolves(0);
 826
 827   Predictor testing_master(true, true);
 828   net::ProxyConfig config = net::ProxyConfig::CreateFromCustomPacURL(GURL(
 829       "http://foopy/proxy.pac"));
 830   testing_master.proxy_service_ = net::ProxyService::CreateFixed(config);
 831
 832   GURL goog("http://www.google.com:80");
 833   testing_master.Resolve(goog, UrlInfo::OMNIBOX_MOTIVATED);
 834
 835   // Proxy may not be in use (the PAC script has not yet been evaluated), so the
 836   // name has been registered for pre-resolve.
 837   EXPECT_FALSE(testing_master.work_queue_.IsEmpty());
 838
 839   delete testing_master.proxy_service_;
 840   testing_master.Shutdown();
 841 }
 842
 843 }  // namespace chrome_browser_net