chrome/browser/net/predictor_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <time.h>
   6
   7 #include <algorithm>
   8 #include <sstream>
   9 #include <string>
  10
  11 #include "base/memory/scoped_ptr.h"
  12 #include "base/message_loop/message_loop.h"
  13 #include "base/strings/string_number_conversions.h"
  14 #include "base/timer/timer.h"
  15 #include "base/values.h"
  16 #include "chrome/browser/net/predictor.h"
  17 #include "chrome/browser/net/url_info.h"
  18 #include "components/network_hints/common/network_hints_common.h"
  19 #include "content/public/test/test_browser_thread.h"
  20 #include "net/base/address_list.h"
  21 #include "net/base/load_flags.h"
  22 #include "net/base/net_errors.h"
  23 #include "net/base/winsock_init.h"
  24 #include "net/dns/mock_host_resolver.h"
  25 #include "net/http/transport_security_state.h"
  26 #include "net/proxy/proxy_config_service_fixed.h"
  27 #include "net/proxy/proxy_service.h"
  28 #include "testing/gmock/include/gmock/gmock.h"
  29 #include "testing/gtest/include/gtest/gtest.h"
  30
  31 using base::Time;
  32 using base::TimeDelta;
  33 using content::BrowserThread;
  34
  35 namespace chrome_browser_net {
  36
  37 class WaitForResolutionHelper;
  38
  39 typedef base::RepeatingTimer<WaitForResolutionHelper> HelperTimer;
  40
  41 class WaitForResolutionHelper {
  42  public:
  43   WaitForResolutionHelper(Predictor* predictor, const UrlList& hosts,
  44                           HelperTimer* timer, int checks_until_quit)
  45       : predictor_(predictor),
  46         hosts_(hosts),
  47         timer_(timer),
  48         checks_until_quit_(checks_until_quit) {
  49   }
  50
  51   void CheckIfResolutionsDone() {
  52     if (--checks_until_quit_ > 0) {
  53       for (UrlList::const_iterator i = hosts_.begin(); i != hosts_.end(); ++i)
  54         if (predictor_->GetResolutionDuration(*i) ==
  55             UrlInfo::NullDuration())
  56           return;  // We don't have resolution for that host.
  57     }
  58
  59     // When all hostnames have been resolved, or we've hit the limit,
  60     // exit the loop.
  61     timer_->Stop();
  62     base::MessageLoop::current()->Quit();
  63     delete timer_;
  64     delete this;
  65   }
  66
  67  private:
  68   Predictor* predictor_;
  69   const UrlList hosts_;
  70   HelperTimer* timer_;
  71   int checks_until_quit_;
  72 };
  73
  74 class PredictorTest : public testing::Test {
  75  public:
  76   PredictorTest()
  77       : ui_thread_(BrowserThread::UI, &loop_),
  78         io_thread_(BrowserThread::IO, &loop_),
  79         host_resolver_(new net::MockCachingHostResolver()) {
  80   }
  81
  82  protected:
  83   void SetUp() override {
  84 #if defined(OS_WIN)
  85     net::EnsureWinsockInit();
  86 #endif
  87     Predictor::set_max_parallel_resolves(
  88         Predictor::kMaxSpeculativeParallelResolves);
  89     Predictor::set_max_queueing_delay(
  90         Predictor::kMaxSpeculativeResolveQueueDelayMs);
  91     // Since we are using a caching HostResolver, the following latencies will
  92     // only be incurred by the first request, after which the result will be
  93     // cached internally by |host_resolver_|.
  94     net::RuleBasedHostResolverProc* rules = host_resolver_->rules();
  95     rules->AddRuleWithLatency("www.google.com", "127.0.0.1", 50);
  96     rules->AddRuleWithLatency("gmail.google.com.com", "127.0.0.1", 70);
  97     rules->AddRuleWithLatency("mail.google.com", "127.0.0.1", 44);
  98     rules->AddRuleWithLatency("gmail.com", "127.0.0.1", 63);
  99   }
 100
 101   void WaitForResolution(Predictor* predictor, const UrlList& hosts) {
 102     HelperTimer* timer = new HelperTimer();
 103     // By default allow the loop to run for a minute -- 600 iterations.
 104     timer->Start(FROM_HERE, TimeDelta::FromMilliseconds(100),
 105                  new WaitForResolutionHelper(predictor, hosts, timer, 600),
 106                  &WaitForResolutionHelper::CheckIfResolutionsDone);
 107     base::MessageLoop::current()->Run();
 108   }
 109
 110   void WaitForResolutionWithLimit(
 111       Predictor* predictor, const UrlList& hosts, int limit) {
 112     HelperTimer* timer = new HelperTimer();
 113     timer->Start(FROM_HERE, TimeDelta::FromMilliseconds(100),
 114                  new WaitForResolutionHelper(predictor, hosts, timer, limit),
 115                  &WaitForResolutionHelper::CheckIfResolutionsDone);
 116     base::MessageLoop::current()->Run();
 117   }
 118
 119  private:
 120   // IMPORTANT: do not move this below |host_resolver_|; the host resolver
 121   // must not outlive the message loop, otherwise bad things can happen
 122   // (like posting to a deleted message loop).
 123   base::MessageLoopForUI loop_;
 124   content::TestBrowserThread ui_thread_;
 125   content::TestBrowserThread io_thread_;
 126
 127  protected:
 128   scoped_ptr<net::MockCachingHostResolver> host_resolver_;
 129 };
 130
 131 //------------------------------------------------------------------------------
 132
 133 TEST_F(PredictorTest, StartupShutdownTest) {
 134   Predictor testing_master(true, true);
 135   testing_master.Shutdown();
 136 }
 137
 138
 139 TEST_F(PredictorTest, ShutdownWhenResolutionIsPendingTest) {
 140   scoped_ptr<net::HostResolver> host_resolver(new net::HangingHostResolver());
 141
 142   Predictor testing_master(true, true);
 143   testing_master.SetHostResolver(host_resolver.get());
 144
 145   GURL localhost("http://localhost:80");
 146   UrlList names;
 147   names.push_back(localhost);
 148
 149   testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 150
 151   base::MessageLoop::current()->PostDelayedTask(
 152       FROM_HERE,
 153       base::MessageLoop::QuitClosure(),
 154       base::TimeDelta::FromMilliseconds(500));
 155   base::MessageLoop::current()->Run();
 156
 157   EXPECT_FALSE(testing_master.WasFound(localhost));
 158
 159   testing_master.Shutdown();
 160
 161   // Clean up after ourselves.
 162   base::MessageLoop::current()->RunUntilIdle();
 163 }
 164
 165 TEST_F(PredictorTest, SingleLookupTest) {
 166   Predictor testing_master(true, true);
 167   testing_master.SetHostResolver(host_resolver_.get());
 168
 169   GURL goog("http://www.google.com:80");
 170
 171   UrlList names;
 172   names.push_back(goog);
 173
 174   // Try to flood the predictor with many concurrent requests.
 175   for (int i = 0; i < 10; i++)
 176     testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 177
 178   WaitForResolution(&testing_master, names);
 179
 180   EXPECT_TRUE(testing_master.WasFound(goog));
 181
 182   base::MessageLoop::current()->RunUntilIdle();
 183
 184   EXPECT_GT(testing_master.peak_pending_lookups(), names.size() / 2);
 185   EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
 186   EXPECT_LE(testing_master.peak_pending_lookups(),
 187             testing_master.max_concurrent_dns_lookups());
 188
 189   testing_master.Shutdown();
 190 }
 191
 192 TEST_F(PredictorTest, ConcurrentLookupTest) {
 193   host_resolver_->rules()->AddSimulatedFailure("*.notfound");
 194
 195   Predictor testing_master(true, true);
 196   testing_master.SetHostResolver(host_resolver_.get());
 197
 198   GURL goog("http://www.google.com:80"),
 199       goog2("http://gmail.google.com.com:80"),
 200       goog3("http://mail.google.com:80"),
 201       goog4("http://gmail.com:80");
 202   GURL bad1("http://bad1.notfound:80"),
 203       bad2("http://bad2.notfound:80");
 204
 205   UrlList names;
 206   names.push_back(goog);
 207   names.push_back(goog3);
 208   names.push_back(bad1);
 209   names.push_back(goog2);
 210   names.push_back(bad2);
 211   names.push_back(goog4);
 212   names.push_back(goog);
 213
 214   // Try to flood the predictor with many concurrent requests.
 215   for (int i = 0; i < 10; i++)
 216     testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 217
 218   WaitForResolution(&testing_master, names);
 219
 220   EXPECT_TRUE(testing_master.WasFound(goog));
 221   EXPECT_TRUE(testing_master.WasFound(goog3));
 222   EXPECT_TRUE(testing_master.WasFound(goog2));
 223   EXPECT_TRUE(testing_master.WasFound(goog4));
 224   EXPECT_FALSE(testing_master.WasFound(bad1));
 225   EXPECT_FALSE(testing_master.WasFound(bad2));
 226
 227   base::MessageLoop::current()->RunUntilIdle();
 228
 229   EXPECT_FALSE(testing_master.WasFound(bad1));
 230   EXPECT_FALSE(testing_master.WasFound(bad2));
 231
 232   EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
 233   EXPECT_LE(testing_master.peak_pending_lookups(),
 234             testing_master.max_concurrent_dns_lookups());
 235
 236   testing_master.Shutdown();
 237 }
 238
 239 TEST_F(PredictorTest, MassiveConcurrentLookupTest) {
 240   host_resolver_->rules()->AddSimulatedFailure("*.notfound");
 241
 242   Predictor testing_master(true, true);
 243   testing_master.SetHostResolver(host_resolver_.get());
 244
 245   UrlList names;
 246   for (int i = 0; i < 100; i++)
 247     names.push_back(GURL(
 248         "http://host" + base::IntToString(i) + ".notfound:80"));
 249
 250   // Try to flood the predictor with many concurrent requests.
 251   for (int i = 0; i < 10; i++)
 252     testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 253
 254   WaitForResolution(&testing_master, names);
 255
 256   base::MessageLoop::current()->RunUntilIdle();
 257
 258   EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
 259   EXPECT_LE(testing_master.peak_pending_lookups(),
 260             testing_master.max_concurrent_dns_lookups());
 261
 262   testing_master.Shutdown();
 263 }
 264
 265 //------------------------------------------------------------------------------
 266 // Functions to help synthesize and test serializations of subresource referrer
 267 // lists.
 268
 269 // Return a motivation_list if we can find one for the given motivating_host (or
 270 // NULL if a match is not found).
 271 static const base::ListValue* FindSerializationMotivation(
 272     const GURL& motivation,
 273     const base::ListValue* referral_list) {
 274   CHECK_LT(0u, referral_list->GetSize());  // Room for version.
 275   int format_version = -1;
 276   CHECK(referral_list->GetInteger(0, &format_version));
 277   CHECK_EQ(Predictor::kPredictorReferrerVersion, format_version);
 278   const base::ListValue* motivation_list(NULL);
 279   for (size_t i = 1; i < referral_list->GetSize(); ++i) {
 280     referral_list->GetList(i, &motivation_list);
 281     std::string existing_spec;
 282     EXPECT_TRUE(motivation_list->GetString(0, &existing_spec));
 283     if (motivation == GURL(existing_spec))
 284       return motivation_list;
 285   }
 286   return NULL;
 287 }
 288
 289 static base::ListValue* FindSerializationMotivation(
 290     const GURL& motivation,
 291     base::ListValue* referral_list) {
 292   return const_cast<base::ListValue*>(FindSerializationMotivation(
 293       motivation, static_cast<const base::ListValue*>(referral_list)));
 294 }
 295
 296 // Create a new empty serialization list.
 297 static base::ListValue* NewEmptySerializationList() {
 298   base::ListValue* list = new base::ListValue;
 299   list->Append(
 300       new base::FundamentalValue(Predictor::kPredictorReferrerVersion));
 301   return list;
 302 }
 303
 304 // Add a motivating_url and a subresource_url to a serialized list, using
 305 // this given latency. This is a helper function for quickly building these
 306 // lists.
 307 static void AddToSerializedList(const GURL& motivation,
 308                                 const GURL& subresource,
 309                                 double use_rate,
 310                                 base::ListValue* referral_list) {
 311   // Find the motivation if it is already used.
 312   base::ListValue* motivation_list = FindSerializationMotivation(motivation,
 313                                                            referral_list);
 314   if (!motivation_list) {
 315     // This is the first mention of this motivation, so build a list.
 316     motivation_list = new base::ListValue;
 317     motivation_list->Append(new base::StringValue(motivation.spec()));
 318     // Provide empty subresource list.
 319     motivation_list->Append(new base::ListValue());
 320
 321     // ...and make it part of the serialized referral_list.
 322     referral_list->Append(motivation_list);
 323   }
 324
 325   base::ListValue* subresource_list(NULL);
 326   // 0 == url; 1 == subresource_list.
 327   EXPECT_TRUE(motivation_list->GetList(1, &subresource_list));
 328
 329   // We won't bother to check for the subresource being there already.  Worst
 330   // case, during deserialization, the latency value we supply plus the
 331   // existing value(s) will be added to the referrer.
 332
 333   subresource_list->Append(new base::StringValue(subresource.spec()));
 334   subresource_list->Append(new base::FundamentalValue(use_rate));
 335 }
 336
 337 // For a given motivation, and subresource, find what latency is currently
 338 // listed.  This assume a well formed serialization, which has at most one such
 339 // entry for any pair of names.  If no such pair is found, then return false.
 340 // Data is written into use_rate arguments.
 341 static bool GetDataFromSerialization(const GURL& motivation,
 342                                      const GURL& subresource,
 343                                      const base::ListValue& referral_list,
 344                                      double* use_rate) {
 345   const base::ListValue* motivation_list =
 346       FindSerializationMotivation(motivation, &referral_list);
 347   if (!motivation_list)
 348     return false;
 349   const base::ListValue* subresource_list;
 350   EXPECT_TRUE(motivation_list->GetList(1, &subresource_list));
 351   for (size_t i = 0; i < subresource_list->GetSize();) {
 352     std::string url_spec;
 353     EXPECT_TRUE(subresource_list->GetString(i++, &url_spec));
 354     EXPECT_TRUE(subresource_list->GetDouble(i++, use_rate));
 355     if (subresource == GURL(url_spec)) {
 356       return true;
 357     }
 358   }
 359   return false;
 360 }
 361
 362 //------------------------------------------------------------------------------
 363
 364 // Make sure nil referral lists really have no entries, and no latency listed.
 365 TEST_F(PredictorTest, ReferrerSerializationNilTest) {
 366   Predictor predictor(true, true);
 367   predictor.SetHostResolver(host_resolver_.get());
 368
 369   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 370   predictor.SerializeReferrers(referral_list.get());
 371   EXPECT_EQ(1U, referral_list->GetSize());
 372   EXPECT_FALSE(GetDataFromSerialization(
 373     GURL("http://a.com:79"), GURL("http://b.com:78"),
 374       *referral_list.get(), NULL));
 375
 376   predictor.Shutdown();
 377 }
 378
 379 // Make sure that when a serialization list includes a value, that it can be
 380 // deserialized into the database, and can be extracted back out via
 381 // serialization without being changed.
 382 TEST_F(PredictorTest, ReferrerSerializationSingleReferrerTest) {
 383   Predictor predictor(true, true);
 384   predictor.SetHostResolver(host_resolver_.get());
 385   const GURL motivation_url("http://www.google.com:91");
 386   const GURL subresource_url("http://icons.google.com:90");
 387   const double kUseRate = 23.4;
 388   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 389
 390   AddToSerializedList(motivation_url, subresource_url,
 391       kUseRate, referral_list.get());
 392
 393   predictor.DeserializeReferrers(*referral_list.get());
 394
 395   base::ListValue recovered_referral_list;
 396   predictor.SerializeReferrers(&recovered_referral_list);
 397   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 398   double rate;
 399   EXPECT_TRUE(GetDataFromSerialization(
 400       motivation_url, subresource_url, recovered_referral_list, &rate));
 401   EXPECT_EQ(rate, kUseRate);
 402
 403   predictor.Shutdown();
 404 }
 405
 406 // Check that GetHtmlReferrerLists() doesn't crash when given duplicated
 407 // domains for referring URL, and that it sorts the results in the
 408 // correct order.
 409 TEST_F(PredictorTest, GetHtmlReferrerLists) {
 410   Predictor predictor(true, true);
 411   predictor.SetHostResolver(host_resolver_.get());
 412   const double kUseRate = 23.4;
 413   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 414
 415   AddToSerializedList(
 416       GURL("http://d.google.com/x1"),
 417       GURL("http://foo.com/"),
 418       kUseRate, referral_list.get());
 419
 420   // Duplicated hostname (d.google.com). This should not cause any crashes
 421   // (i.e. crbug.com/116345)
 422   AddToSerializedList(
 423       GURL("http://d.google.com/x2"),
 424       GURL("http://foo.com/"),
 425       kUseRate, referral_list.get());
 426
 427   AddToSerializedList(
 428       GURL("http://a.yahoo.com/y"),
 429       GURL("http://foo1.com/"),
 430       kUseRate, referral_list.get());
 431
 432   AddToSerializedList(
 433       GURL("http://b.google.com/x3"),
 434       GURL("http://foo2.com/"),
 435       kUseRate, referral_list.get());
 436
 437   AddToSerializedList(
 438       GURL("http://d.yahoo.com/x5"),
 439       GURL("http://i.like.turtles/"),
 440       kUseRate, referral_list.get());
 441
 442   AddToSerializedList(
 443       GURL("http://c.yahoo.com/x4"),
 444       GURL("http://foo3.com/"),
 445       kUseRate, referral_list.get());
 446
 447   predictor.DeserializeReferrers(*referral_list.get());
 448
 449   std::string html;
 450   predictor.GetHtmlReferrerLists(&html);
 451
 452   // The lexicographic sorting of hostnames would be:
 453   //   a.yahoo.com
 454   //   b.google.com
 455   //   c.yahoo.com
 456   //   d.google.com
 457   //   d.yahoo.com
 458   //
 459   // However we expect to sort them by domain in the output:
 460   //   b.google.com
 461   //   d.google.com
 462   //   a.yahoo.com
 463   //   c.yahoo.com
 464   //   d.yahoo.com
 465
 466   size_t pos[] = {
 467       html.find("<td rowspan=1>http://b.google.com/x3"),
 468       html.find("<td rowspan=1>http://d.google.com/x1"),
 469       html.find("<td rowspan=1>http://d.google.com/x2"),
 470       html.find("<td rowspan=1>http://a.yahoo.com/y"),
 471       html.find("<td rowspan=1>http://c.yahoo.com/x4"),
 472       html.find("<td rowspan=1>http://d.yahoo.com/x5"),
 473   };
 474
 475   // Make sure things appeared in the expected order.
 476   for (size_t i = 1; i < arraysize(pos); ++i) {
 477     EXPECT_LT(pos[i - 1], pos[i]) << "Mismatch for pos[" << i << "]";
 478   }
 479
 480   predictor.Shutdown();
 481 }
 482
 483 // Verify that two floats are within 1% of each other in value.
 484 #define EXPECT_SIMILAR(a, b) do { \
 485     double espilon_ratio = 1.01;  \
 486     if ((a) < 0.)  \
 487       espilon_ratio = 1 / espilon_ratio;  \
 488     EXPECT_LT(a, espilon_ratio * (b));   \
 489     EXPECT_GT((a) * espilon_ratio, b);   \
 490     } while (0)
 491
 492
 493 // Make sure the Trim() functionality works as expected.
 494 TEST_F(PredictorTest, ReferrerSerializationTrimTest) {
 495   Predictor predictor(true, true);
 496   predictor.SetHostResolver(host_resolver_.get());
 497   GURL motivation_url("http://www.google.com:110");
 498
 499   GURL icon_subresource_url("http://icons.google.com:111");
 500   const double kRateIcon = 16.0 * Predictor::kDiscardableExpectedValue;
 501   GURL img_subresource_url("http://img.google.com:118");
 502   const double kRateImg = 8.0 * Predictor::kDiscardableExpectedValue;
 503
 504   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 505   AddToSerializedList(
 506       motivation_url, icon_subresource_url, kRateIcon, referral_list.get());
 507   AddToSerializedList(
 508       motivation_url, img_subresource_url, kRateImg, referral_list.get());
 509
 510   predictor.DeserializeReferrers(*referral_list.get());
 511
 512   base::ListValue recovered_referral_list;
 513   predictor.SerializeReferrers(&recovered_referral_list);
 514   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 515   double rate;
 516   EXPECT_TRUE(GetDataFromSerialization(
 517       motivation_url, icon_subresource_url, recovered_referral_list,
 518       &rate));
 519   EXPECT_SIMILAR(rate, kRateIcon);
 520
 521   EXPECT_TRUE(GetDataFromSerialization(
 522       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 523   EXPECT_SIMILAR(rate, kRateImg);
 524
 525   // Each time we Trim 24 times, the user_rate figures should reduce by a factor
 526   // of two,  until they are small, and then a trim will delete the whole entry.
 527   for (int i = 0; i < 24; ++i)
 528     predictor.TrimReferrersNow();
 529   predictor.SerializeReferrers(&recovered_referral_list);
 530   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 531   EXPECT_TRUE(GetDataFromSerialization(
 532       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 533   EXPECT_SIMILAR(rate, kRateIcon / 2);
 534
 535   EXPECT_TRUE(GetDataFromSerialization(
 536       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 537   EXPECT_SIMILAR(rate, kRateImg / 2);
 538
 539   for (int i = 0; i < 24; ++i)
 540     predictor.TrimReferrersNow();
 541   predictor.SerializeReferrers(&recovered_referral_list);
 542   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 543   EXPECT_TRUE(GetDataFromSerialization(
 544       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 545   EXPECT_SIMILAR(rate, kRateIcon / 4);
 546   EXPECT_TRUE(GetDataFromSerialization(
 547       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 548   EXPECT_SIMILAR(rate, kRateImg / 4);
 549
 550   for (int i = 0; i < 24; ++i)
 551     predictor.TrimReferrersNow();
 552   predictor.SerializeReferrers(&recovered_referral_list);
 553   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 554   EXPECT_TRUE(GetDataFromSerialization(
 555       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 556   EXPECT_SIMILAR(rate, kRateIcon / 8);
 557
 558   // Img is below threshold, and so it gets deleted.
 559   EXPECT_FALSE(GetDataFromSerialization(
 560       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 561
 562   for (int i = 0; i < 24; ++i)
 563     predictor.TrimReferrersNow();
 564   predictor.SerializeReferrers(&recovered_referral_list);
 565   // Icon is also trimmed away, so entire set gets discarded.
 566   EXPECT_EQ(1U, recovered_referral_list.GetSize());
 567   EXPECT_FALSE(GetDataFromSerialization(
 568       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 569   EXPECT_FALSE(GetDataFromSerialization(
 570       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 571
 572   predictor.Shutdown();
 573 }
 574
 575
 576 TEST_F(PredictorTest, PriorityQueuePushPopTest) {
 577   Predictor::HostNameQueue queue;
 578
 579   GURL first("http://first:80"), second("http://second:90");
 580
 581   // First check high priority queue FIFO functionality.
 582   EXPECT_TRUE(queue.IsEmpty());
 583   queue.Push(first, UrlInfo::LEARNED_REFERAL_MOTIVATED);
 584   EXPECT_FALSE(queue.IsEmpty());
 585   queue.Push(second, UrlInfo::MOUSE_OVER_MOTIVATED);
 586   EXPECT_FALSE(queue.IsEmpty());
 587   EXPECT_EQ(queue.Pop(), first);
 588   EXPECT_FALSE(queue.IsEmpty());
 589   EXPECT_EQ(queue.Pop(), second);
 590   EXPECT_TRUE(queue.IsEmpty());
 591
 592   // Then check low priority queue FIFO functionality.
 593   queue.Push(first, UrlInfo::PAGE_SCAN_MOTIVATED);
 594   EXPECT_FALSE(queue.IsEmpty());
 595   queue.Push(second, UrlInfo::OMNIBOX_MOTIVATED);
 596   EXPECT_FALSE(queue.IsEmpty());
 597   EXPECT_EQ(queue.Pop(), first);
 598   EXPECT_FALSE(queue.IsEmpty());
 599   EXPECT_EQ(queue.Pop(), second);
 600   EXPECT_TRUE(queue.IsEmpty());
 601 }
 602
 603 TEST_F(PredictorTest, PriorityQueueReorderTest) {
 604   Predictor::HostNameQueue queue;
 605
 606   // Push all the low priority items.
 607   GURL low1("http://low1:80"),
 608       low2("http://low2:80"),
 609       low3("http://low3:443"),
 610       low4("http://low4:80"),
 611       low5("http://low5:80"),
 612       hi1("http://hi1:80"),
 613       hi2("http://hi2:80"),
 614       hi3("http://hi3:80");
 615
 616   EXPECT_TRUE(queue.IsEmpty());
 617   queue.Push(low1, UrlInfo::PAGE_SCAN_MOTIVATED);
 618   queue.Push(low2, UrlInfo::UNIT_TEST_MOTIVATED);
 619   queue.Push(low3, UrlInfo::LINKED_MAX_MOTIVATED);
 620   queue.Push(low4, UrlInfo::OMNIBOX_MOTIVATED);
 621   queue.Push(low5, UrlInfo::STARTUP_LIST_MOTIVATED);
 622   queue.Push(low4, UrlInfo::OMNIBOX_MOTIVATED);
 623
 624   // Push all the high prority items
 625   queue.Push(hi1, UrlInfo::LEARNED_REFERAL_MOTIVATED);
 626   queue.Push(hi2, UrlInfo::STATIC_REFERAL_MOTIVATED);
 627   queue.Push(hi3, UrlInfo::MOUSE_OVER_MOTIVATED);
 628
 629   // Check that high priority stuff comes out first, and in FIFO order.
 630   EXPECT_EQ(queue.Pop(), hi1);
 631   EXPECT_EQ(queue.Pop(), hi2);
 632   EXPECT_EQ(queue.Pop(), hi3);
 633
 634   // ...and then low priority strings.
 635   EXPECT_EQ(queue.Pop(), low1);
 636   EXPECT_EQ(queue.Pop(), low2);
 637   EXPECT_EQ(queue.Pop(), low3);
 638   EXPECT_EQ(queue.Pop(), low4);
 639   EXPECT_EQ(queue.Pop(), low5);
 640   EXPECT_EQ(queue.Pop(), low4);
 641
 642   EXPECT_TRUE(queue.IsEmpty());
 643 }
 644
 645 TEST_F(PredictorTest, CanonicalizeUrl) {
 646   // Base case, only handles HTTP and HTTPS.
 647   EXPECT_EQ(GURL(), Predictor::CanonicalizeUrl(GURL("ftp://anything")));
 648
 649   // Remove path testing.
 650   GURL long_url("http://host:999/path?query=value");
 651   EXPECT_EQ(Predictor::CanonicalizeUrl(long_url), long_url.GetWithEmptyPath());
 652
 653   // Default port cannoncalization.
 654   GURL implied_port("http://test");
 655   GURL explicit_port("http://test:80");
 656   EXPECT_EQ(Predictor::CanonicalizeUrl(implied_port),
 657             Predictor::CanonicalizeUrl(explicit_port));
 658
 659   // Port is still maintained.
 660   GURL port_80("http://test:80");
 661   GURL port_90("http://test:90");
 662   EXPECT_NE(Predictor::CanonicalizeUrl(port_80),
 663             Predictor::CanonicalizeUrl(port_90));
 664
 665   // Host is still maintained.
 666   GURL host_1("http://test_1");
 667   GURL host_2("http://test_2");
 668   EXPECT_NE(Predictor::CanonicalizeUrl(host_1),
 669             Predictor::CanonicalizeUrl(host_2));
 670
 671   // Scheme is maintained (mismatch identified).
 672   GURL http("http://test");
 673   GURL https("https://test");
 674   EXPECT_NE(Predictor::CanonicalizeUrl(http),
 675             Predictor::CanonicalizeUrl(https));
 676
 677   // Https works fine.
 678   GURL long_https("https://host:999/path?query=value");
 679   EXPECT_EQ(Predictor::CanonicalizeUrl(long_https),
 680             long_https.GetWithEmptyPath());
 681 }
 682
 683 TEST_F(PredictorTest, DiscardPredictorResults) {
 684   SimplePredictor predictor(true, true);
 685   predictor.SetHostResolver(host_resolver_.get());
 686   base::ListValue referral_list;
 687   predictor.SerializeReferrers(&referral_list);
 688   EXPECT_EQ(1U, referral_list.GetSize());
 689
 690   GURL host_1("http://test_1");
 691   GURL host_2("http://test_2");
 692   predictor.LearnFromNavigation(host_1, host_2);
 693
 694   predictor.SerializeReferrers(&referral_list);
 695   EXPECT_EQ(2U, referral_list.GetSize());
 696
 697   predictor.DiscardAllResults();
 698   predictor.SerializeReferrers(&referral_list);
 699   EXPECT_EQ(1U, referral_list.GetSize());
 700
 701   predictor.Shutdown();
 702 }
 703
 704 class TestPredictorObserver : public PredictorObserver {
 705  public:
 706   // PredictorObserver implementation:
 707   void OnPreconnectUrl(const GURL& url,
 708                        const GURL& first_party_for_cookies,
 709                        UrlInfo::ResolutionMotivation motivation,
 710                        int count) override {
 711     preconnected_urls_.push_back(url);
 712   }
 713
 714   std::vector<GURL> preconnected_urls_;
 715 };
 716
 717 // Tests that preconnects apply the HSTS list.
 718 TEST_F(PredictorTest, HSTSRedirect) {
 719   const GURL kHttpUrl("http://example.com");
 720   const GURL kHttpsUrl("https://example.com");
 721
 722   const base::Time expiry =
 723       base::Time::Now() + base::TimeDelta::FromSeconds(1000);
 724   net::TransportSecurityState state;
 725   state.AddHSTS(kHttpUrl.host(), expiry, false);
 726
 727   Predictor predictor(true, true);
 728   TestPredictorObserver observer;
 729   predictor.SetObserver(&observer);
 730   predictor.SetTransportSecurityState(&state);
 731
 732   predictor.PreconnectUrl(kHttpUrl, GURL(), UrlInfo::OMNIBOX_MOTIVATED, 2);
 733   ASSERT_EQ(1u, observer.preconnected_urls_.size());
 734   EXPECT_EQ(kHttpsUrl, observer.preconnected_urls_[0]);
 735
 736   predictor.Shutdown();
 737 }
 738
 739 // Tests that preconnecting a URL on the HSTS list preconnects the subresources
 740 // for the SSL version.
 741 TEST_F(PredictorTest, HSTSRedirectSubresources) {
 742   const GURL kHttpUrl("http://example.com");
 743   const GURL kHttpsUrl("https://example.com");
 744   const GURL kSubresourceUrl("https://images.example.com");
 745   const double kUseRate = 23.4;
 746
 747   const base::Time expiry =
 748       base::Time::Now() + base::TimeDelta::FromSeconds(1000);
 749   net::TransportSecurityState state;
 750   state.AddHSTS(kHttpUrl.host(), expiry, false);
 751
 752   SimplePredictor predictor(true, true);
 753   TestPredictorObserver observer;
 754   predictor.SetObserver(&observer);
 755   predictor.SetTransportSecurityState(&state);
 756
 757   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 758   AddToSerializedList(
 759       kHttpsUrl, kSubresourceUrl, kUseRate, referral_list.get());
 760   predictor.DeserializeReferrers(*referral_list.get());
 761
 762   predictor.PreconnectUrlAndSubresources(kHttpUrl, GURL());
 763   ASSERT_EQ(2u, observer.preconnected_urls_.size());
 764   EXPECT_EQ(kHttpsUrl, observer.preconnected_urls_[0]);
 765   EXPECT_EQ(kSubresourceUrl, observer.preconnected_urls_[1]);
 766
 767   predictor.Shutdown();
 768 }
 769
 770 TEST_F(PredictorTest, NoProxyService) {
 771   // Don't actually try to resolve names.
 772   Predictor::set_max_parallel_resolves(0);
 773
 774   Predictor testing_master(true, true);
 775
 776   GURL goog("http://www.google.com:80");
 777   testing_master.Resolve(goog, UrlInfo::OMNIBOX_MOTIVATED);
 778   EXPECT_FALSE(testing_master.work_queue_.IsEmpty());
 779
 780   testing_master.Shutdown();
 781 }
 782
 783 TEST_F(PredictorTest, ProxyDefinitelyEnabled) {
 784   // Don't actually try to resolve names.
 785   Predictor::set_max_parallel_resolves(0);
 786
 787   Predictor testing_master(true, true);
 788
 789   net::ProxyConfig config;
 790   config.proxy_rules().ParseFromString("http=socks://localhost:12345");
 791   testing_master.proxy_service_ = net::ProxyService::CreateFixed(config);
 792
 793   GURL goog("http://www.google.com:80");
 794   testing_master.Resolve(goog, UrlInfo::OMNIBOX_MOTIVATED);
 795
 796   // Proxy is definitely in use, so there is no need to pre-resolve the domain.
 797   EXPECT_TRUE(testing_master.work_queue_.IsEmpty());
 798
 799   delete testing_master.proxy_service_;
 800   testing_master.Shutdown();
 801 }
 802
 803 TEST_F(PredictorTest, ProxyDefinitelyNotEnabled) {
 804   // Don't actually try to resolve names.
 805   Predictor::set_max_parallel_resolves(0);
 806
 807   Predictor testing_master(true, true);
 808   net::ProxyConfig config = net::ProxyConfig::CreateDirect();
 809   testing_master.proxy_service_ = net::ProxyService::CreateFixed(config);
 810
 811   GURL goog("http://www.google.com:80");
 812   testing_master.Resolve(goog, UrlInfo::OMNIBOX_MOTIVATED);
 813
 814   // Proxy is not in use, so the name has been registered for pre-resolve.
 815   EXPECT_FALSE(testing_master.work_queue_.IsEmpty());
 816
 817   delete testing_master.proxy_service_;
 818   testing_master.Shutdown();
 819 }
 820
 821 TEST_F(PredictorTest, ProxyMaybeEnabled) {
 822   // Don't actually try to resolve names.
 823   Predictor::set_max_parallel_resolves(0);
 824
 825   Predictor testing_master(true, true);
 826   net::ProxyConfig config = net::ProxyConfig::CreateFromCustomPacURL(GURL(
 827       "http://foopy/proxy.pac"));
 828   testing_master.proxy_service_ = net::ProxyService::CreateFixed(config);
 829
 830   GURL goog("http://www.google.com:80");
 831   testing_master.Resolve(goog, UrlInfo::OMNIBOX_MOTIVATED);
 832
 833   // Proxy may not be in use (the PAC script has not yet been evaluated), so the
 834   // name has been registered for pre-resolve.
 835   EXPECT_FALSE(testing_master.work_queue_.IsEmpty());
 836
 837   delete testing_master.proxy_service_;
 838   testing_master.Shutdown();
 839 }
 840
 841 }  // namespace chrome_browser_net