google_apis/gcm/engine/connection_factory_impl_unittest.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "google_apis/gcm/engine/connection_factory_impl.h"
   6
   7 #include <cmath>
   8
   9 #include "base/message_loop/message_loop.h"
  10 #include "base/run_loop.h"
  11 #include "base/test/simple_test_tick_clock.h"
  12 #include "google_apis/gcm/base/mcs_util.h"
  13 #include "google_apis/gcm/engine/fake_connection_handler.h"
  14 #include "google_apis/gcm/monitoring/fake_gcm_stats_recorder.h"
  15 #include "net/base/backoff_entry.h"
  16 #include "net/http/http_network_session.h"
  17 #include "testing/gtest/include/gtest/gtest.h"
  18
  19 class Policy;
  20
  21 namespace gcm {
  22 namespace {
  23
  24 const char kMCSEndpoint[] = "http://my.server";
  25 const char kMCSEndpoint2[] = "http://my.alt.server";
  26
  27 const int kBackoffDelayMs = 1;
  28 const int kBackoffMultiplier = 2;
  29
  30 // A backoff policy with small enough delays that tests aren't burdened.
  31 const net::BackoffEntry::Policy kTestBackoffPolicy = {
  32   // Number of initial errors (in sequence) to ignore before applying
  33   // exponential back-off rules.
  34   0,
  35
  36   // Initial delay for exponential back-off in ms.
  37   kBackoffDelayMs,
  38
  39   // Factor by which the waiting time will be multiplied.
  40   kBackoffMultiplier,
  41
  42   // Fuzzing percentage. ex: 10% will spread requests randomly
  43   // between 90%-100% of the calculated time.
  44   0,
  45
  46   // Maximum amount of time we are willing to delay our request in ms.
  47   10,
  48
  49   // Time to keep an entry from being discarded even when it
  50   // has no significant state, -1 to never discard.
  51   -1,
  52
  53   // Don't use initial delay unless the last request was an error.
  54   false,
  55 };
  56
  57 std::vector<GURL> BuildEndpoints() {
  58   std::vector<GURL> endpoints;
  59   endpoints.push_back(GURL(kMCSEndpoint));
  60   endpoints.push_back(GURL(kMCSEndpoint2));
  61   return endpoints;
  62 }
  63
  64 // Helper for calculating total expected exponential backoff delay given an
  65 // arbitrary number of failed attempts. See BackoffEntry::CalculateReleaseTime.
  66 double CalculateBackoff(int num_attempts) {
  67   double delay = kBackoffDelayMs;
  68   for (int i = 1; i < num_attempts; ++i) {
  69     delay += kBackoffDelayMs * pow(static_cast<double>(kBackoffMultiplier),
  70                                    i - 1);
  71   }
  72   DVLOG(1) << "Expected backoff " << delay << " milliseconds.";
  73   return delay;
  74 }
  75
  76 void ReadContinuation(
  77     scoped_ptr<google::protobuf::MessageLite> message) {
  78 }
  79
  80 void WriteContinuation() {
  81 }
  82
  83 class TestBackoffEntry : public net::BackoffEntry {
  84  public:
  85   explicit TestBackoffEntry(base::SimpleTestTickClock* tick_clock);
  86   virtual ~TestBackoffEntry();
  87
  88   virtual base::TimeTicks ImplGetTimeNow() const OVERRIDE;
  89
  90  private:
  91   base::SimpleTestTickClock* tick_clock_;
  92 };
  93
  94 TestBackoffEntry::TestBackoffEntry(base::SimpleTestTickClock* tick_clock)
  95     : BackoffEntry(&kTestBackoffPolicy),
  96       tick_clock_(tick_clock) {
  97 }
  98
  99 TestBackoffEntry::~TestBackoffEntry() {}
 100
 101 base::TimeTicks TestBackoffEntry::ImplGetTimeNow() const {
 102   return tick_clock_->NowTicks();
 103 }
 104
 105 // A connection factory that stubs out network requests and overrides the
 106 // backoff policy.
 107 class TestConnectionFactoryImpl : public ConnectionFactoryImpl {
 108  public:
 109   TestConnectionFactoryImpl(const base::Closure& finished_callback);
 110   virtual ~TestConnectionFactoryImpl();
 111
 112   void InitializeFactory();
 113
 114   // Overridden stubs.
 115   virtual void ConnectImpl() OVERRIDE;
 116   virtual void InitHandler() OVERRIDE;
 117   virtual scoped_ptr<net::BackoffEntry> CreateBackoffEntry(
 118       const net::BackoffEntry::Policy* const policy) OVERRIDE;
 119   virtual scoped_ptr<ConnectionHandler> CreateConnectionHandler(
 120       base::TimeDelta read_timeout,
 121       const ConnectionHandler::ProtoReceivedCallback& read_callback,
 122       const ConnectionHandler::ProtoSentCallback& write_callback,
 123       const ConnectionHandler::ConnectionChangedCallback& connection_callback)
 124           OVERRIDE;
 125   virtual base::TimeTicks NowTicks() OVERRIDE;
 126
 127   // Helpers for verifying connection attempts are made. Connection results
 128   // must be consumed.
 129   void SetConnectResult(int connect_result);
 130   void SetMultipleConnectResults(int connect_result, int num_expected_attempts);
 131
 132   // Force a login handshake to be delayed.
 133   void SetDelayLogin(bool delay_login);
 134
 135   base::SimpleTestTickClock* tick_clock() { return &tick_clock_; }
 136
 137  private:
 138   // Clock for controlling delay.
 139   base::SimpleTestTickClock tick_clock_;
 140   // The result to return on the next connect attempt.
 141   int connect_result_;
 142   // The number of expected connection attempts;
 143   int num_expected_attempts_;
 144   // Whether all expected connection attempts have been fulfilled since an
 145   // expectation was last set.
 146   bool connections_fulfilled_;
 147   // Whether to delay a login handshake completion or not.
 148   bool delay_login_;
 149   // Callback to invoke when all connection attempts have been made.
 150   base::Closure finished_callback_;
 151   // The current fake connection handler..
 152   FakeConnectionHandler* fake_handler_;
 153   FakeGCMStatsRecorder dummy_recorder_;
 154 };
 155
 156 TestConnectionFactoryImpl::TestConnectionFactoryImpl(
 157     const base::Closure& finished_callback)
 158     : ConnectionFactoryImpl(BuildEndpoints(),
 159                             net::BackoffEntry::Policy(),
 160                             NULL,
 161                             NULL,
 162                             NULL,
 163                             &dummy_recorder_),
 164       connect_result_(net::ERR_UNEXPECTED),
 165       num_expected_attempts_(0),
 166       connections_fulfilled_(true),
 167       delay_login_(false),
 168       finished_callback_(finished_callback),
 169       fake_handler_(NULL) {
 170   // Set a non-null time.
 171   tick_clock_.Advance(base::TimeDelta::FromMilliseconds(1));
 172 }
 173
 174 TestConnectionFactoryImpl::~TestConnectionFactoryImpl() {
 175   EXPECT_EQ(0, num_expected_attempts_);
 176 }
 177
 178 void TestConnectionFactoryImpl::ConnectImpl() {
 179   ASSERT_GT(num_expected_attempts_, 0);
 180   scoped_ptr<mcs_proto::LoginRequest> request(BuildLoginRequest(0, 0, ""));
 181   GetConnectionHandler()->Init(*request, NULL);
 182   OnConnectDone(connect_result_);
 183   if (!NextRetryAttempt().is_null()) {
 184     // Advance the time to the next retry time.
 185     base::TimeDelta time_till_retry =
 186         NextRetryAttempt() - tick_clock_.NowTicks();
 187     tick_clock_.Advance(time_till_retry);
 188   }
 189   --num_expected_attempts_;
 190   if (num_expected_attempts_ == 0) {
 191     connect_result_ = net::ERR_UNEXPECTED;
 192     connections_fulfilled_ = true;
 193     finished_callback_.Run();
 194   }
 195 }
 196
 197 void TestConnectionFactoryImpl::InitHandler() {
 198   EXPECT_NE(connect_result_, net::ERR_UNEXPECTED);
 199   if (!delay_login_)
 200     ConnectionHandlerCallback(net::OK);
 201 }
 202
 203 scoped_ptr<net::BackoffEntry> TestConnectionFactoryImpl::CreateBackoffEntry(
 204     const net::BackoffEntry::Policy* const policy) {
 205   return scoped_ptr<net::BackoffEntry>(new TestBackoffEntry(&tick_clock_));
 206 }
 207
 208 scoped_ptr<ConnectionHandler>
 209 TestConnectionFactoryImpl::CreateConnectionHandler(
 210     base::TimeDelta read_timeout,
 211     const ConnectionHandler::ProtoReceivedCallback& read_callback,
 212     const ConnectionHandler::ProtoSentCallback& write_callback,
 213     const ConnectionHandler::ConnectionChangedCallback& connection_callback) {
 214   fake_handler_ = new FakeConnectionHandler(
 215       base::Bind(&ReadContinuation),
 216       base::Bind(&WriteContinuation));
 217   return make_scoped_ptr<ConnectionHandler>(fake_handler_);
 218 }
 219
 220 base::TimeTicks TestConnectionFactoryImpl::NowTicks() {
 221   return tick_clock_.NowTicks();
 222 }
 223
 224 void TestConnectionFactoryImpl::SetConnectResult(int connect_result) {
 225   DCHECK_NE(connect_result, net::ERR_UNEXPECTED);
 226   ASSERT_EQ(0, num_expected_attempts_);
 227   connections_fulfilled_ = false;
 228   connect_result_ = connect_result;
 229   num_expected_attempts_ = 1;
 230   fake_handler_->ExpectOutgoingMessage(
 231       MCSMessage(kLoginRequestTag,
 232                  BuildLoginRequest(0, 0, "").PassAs<
 233                      const google::protobuf::MessageLite>()));
 234 }
 235
 236 void TestConnectionFactoryImpl::SetMultipleConnectResults(
 237     int connect_result,
 238     int num_expected_attempts) {
 239   DCHECK_NE(connect_result, net::ERR_UNEXPECTED);
 240   DCHECK_GT(num_expected_attempts, 0);
 241   ASSERT_EQ(0, num_expected_attempts_);
 242   connections_fulfilled_ = false;
 243   connect_result_ = connect_result;
 244   num_expected_attempts_ = num_expected_attempts;
 245   for (int i = 0 ; i < num_expected_attempts; ++i) {
 246     fake_handler_->ExpectOutgoingMessage(
 247         MCSMessage(kLoginRequestTag,
 248                    BuildLoginRequest(0, 0, "").PassAs<
 249                        const google::protobuf::MessageLite>()));
 250   }
 251 }
 252
 253 void TestConnectionFactoryImpl::SetDelayLogin(bool delay_login) {
 254   delay_login_ = delay_login;
 255   fake_handler_->set_fail_login(delay_login_);
 256 }
 257
 258 }  // namespace
 259
 260 class ConnectionFactoryImplTest
 261     : public testing::Test,
 262       public ConnectionFactory::ConnectionListener {
 263  public:
 264   ConnectionFactoryImplTest();
 265   virtual ~ConnectionFactoryImplTest();
 266
 267   TestConnectionFactoryImpl* factory() { return &factory_; }
 268   GURL& connected_server() { return connected_server_; }
 269
 270   void WaitForConnections();
 271
 272   // ConnectionFactory::ConnectionListener
 273   virtual void OnConnected(const GURL& current_server,
 274                            const net::IPEndPoint& ip_endpoint) OVERRIDE;
 275   virtual void OnDisconnected() OVERRIDE;
 276
 277  private:
 278   void ConnectionsComplete();
 279
 280   TestConnectionFactoryImpl factory_;
 281   base::MessageLoop message_loop_;
 282   scoped_ptr<base::RunLoop> run_loop_;
 283
 284   GURL connected_server_;
 285 };
 286
 287 ConnectionFactoryImplTest::ConnectionFactoryImplTest()
 288    : factory_(base::Bind(&ConnectionFactoryImplTest::ConnectionsComplete,
 289                          base::Unretained(this))),
 290      run_loop_(new base::RunLoop()) {
 291   factory()->SetConnectionListener(this);
 292   factory()->Initialize(
 293       ConnectionFactory::BuildLoginRequestCallback(),
 294       ConnectionHandler::ProtoReceivedCallback(),
 295       ConnectionHandler::ProtoSentCallback());
 296 }
 297 ConnectionFactoryImplTest::~ConnectionFactoryImplTest() {}
 298
 299 void ConnectionFactoryImplTest::WaitForConnections() {
 300   run_loop_->Run();
 301   run_loop_.reset(new base::RunLoop());
 302 }
 303
 304 void ConnectionFactoryImplTest::ConnectionsComplete() {
 305   if (!run_loop_)
 306     return;
 307   run_loop_->Quit();
 308 }
 309
 310 void ConnectionFactoryImplTest::OnConnected(
 311     const GURL& current_server,
 312     const net::IPEndPoint& ip_endpoint) {
 313   connected_server_ = current_server;
 314 }
 315
 316 void ConnectionFactoryImplTest::OnDisconnected() {
 317   connected_server_ = GURL();
 318 }
 319
 320 // Verify building a connection handler works.
 321 TEST_F(ConnectionFactoryImplTest, Initialize) {
 322   ConnectionHandler* handler = factory()->GetConnectionHandler();
 323   ASSERT_TRUE(handler);
 324   EXPECT_FALSE(factory()->IsEndpointReachable());
 325   EXPECT_FALSE(connected_server().is_valid());
 326 }
 327
 328 // An initial successful connection should not result in backoff.
 329 TEST_F(ConnectionFactoryImplTest, ConnectSuccess) {
 330   factory()->SetConnectResult(net::OK);
 331   factory()->Connect();
 332   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
 333   EXPECT_EQ(factory()->GetCurrentEndpoint(), BuildEndpoints()[0]);
 334   EXPECT_TRUE(factory()->IsEndpointReachable());
 335   EXPECT_TRUE(connected_server().is_valid());
 336 }
 337
 338 // A connection failure should result in backoff, and attempting the fallback
 339 // endpoint next.
 340 TEST_F(ConnectionFactoryImplTest, ConnectFail) {
 341   factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
 342   factory()->Connect();
 343   EXPECT_FALSE(factory()->NextRetryAttempt().is_null());
 344   EXPECT_EQ(factory()->GetCurrentEndpoint(), BuildEndpoints()[1]);
 345   EXPECT_FALSE(factory()->IsEndpointReachable());
 346   EXPECT_FALSE(connected_server().is_valid());
 347 }
 348
 349 // A connection success after a failure should reset backoff.
 350 TEST_F(ConnectionFactoryImplTest, FailThenSucceed) {
 351   factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
 352   base::TimeTicks connect_time = factory()->tick_clock()->NowTicks();
 353   factory()->Connect();
 354   WaitForConnections();
 355   EXPECT_FALSE(factory()->IsEndpointReachable());
 356   EXPECT_FALSE(connected_server().is_valid());
 357   base::TimeTicks retry_time = factory()->NextRetryAttempt();
 358   EXPECT_FALSE(retry_time.is_null());
 359   EXPECT_GE((retry_time - connect_time).InMilliseconds(), CalculateBackoff(1));
 360   factory()->SetConnectResult(net::OK);
 361   WaitForConnections();
 362   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
 363   EXPECT_TRUE(factory()->IsEndpointReachable());
 364   EXPECT_TRUE(connected_server().is_valid());
 365 }
 366
 367 // Multiple connection failures should retry with an exponentially increasing
 368 // backoff, then reset on success.
 369 TEST_F(ConnectionFactoryImplTest, MultipleFailuresThenSucceed) {
 370   const int kNumAttempts = 5;
 371   factory()->SetMultipleConnectResults(net::ERR_CONNECTION_FAILED,
 372                                        kNumAttempts);
 373
 374   base::TimeTicks connect_time = factory()->tick_clock()->NowTicks();
 375   factory()->Connect();
 376   WaitForConnections();
 377   EXPECT_FALSE(factory()->IsEndpointReachable());
 378   EXPECT_FALSE(connected_server().is_valid());
 379   base::TimeTicks retry_time = factory()->NextRetryAttempt();
 380   EXPECT_FALSE(retry_time.is_null());
 381   EXPECT_GE((retry_time - connect_time).InMilliseconds(),
 382             CalculateBackoff(kNumAttempts));
 383
 384   factory()->SetConnectResult(net::OK);
 385   WaitForConnections();
 386   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
 387   EXPECT_TRUE(factory()->IsEndpointReachable());
 388   EXPECT_TRUE(connected_server().is_valid());
 389 }
 390
 391 // Network change events should trigger canary connections.
 392 TEST_F(ConnectionFactoryImplTest, FailThenNetworkChangeEvent) {
 393   factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
 394   factory()->Connect();
 395   WaitForConnections();
 396   base::TimeTicks initial_backoff = factory()->NextRetryAttempt();
 397   EXPECT_FALSE(initial_backoff.is_null());
 398
 399   factory()->SetConnectResult(net::ERR_FAILED);
 400   factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_WIFI);
 401   WaitForConnections();
 402
 403   // Backoff should increase.
 404   base::TimeTicks next_backoff = factory()->NextRetryAttempt();
 405   EXPECT_GT(next_backoff, initial_backoff);
 406   EXPECT_FALSE(factory()->IsEndpointReachable());
 407 }
 408
 409 // Verify that we reconnect even if a canary succeeded then disconnected while
 410 // a backoff was pending.
 411 TEST_F(ConnectionFactoryImplTest, CanarySucceedsThenDisconnects) {
 412   factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
 413   factory()->Connect();
 414   WaitForConnections();
 415   base::TimeTicks initial_backoff = factory()->NextRetryAttempt();
 416   EXPECT_FALSE(initial_backoff.is_null());
 417
 418   factory()->SetConnectResult(net::OK);
 419   factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_ETHERNET);
 420   WaitForConnections();
 421   EXPECT_TRUE(factory()->IsEndpointReachable());
 422   EXPECT_TRUE(connected_server().is_valid());
 423
 424   factory()->SetConnectResult(net::OK);
 425   factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
 426   EXPECT_FALSE(factory()->IsEndpointReachable());
 427   EXPECT_FALSE(connected_server().is_valid());
 428   WaitForConnections();
 429   EXPECT_TRUE(factory()->IsEndpointReachable());
 430   EXPECT_TRUE(connected_server().is_valid());
 431 }
 432
 433 // Verify that if a canary connects, but hasn't finished the handshake, a
 434 // pending backoff attempt doesn't interrupt the connection.
 435 TEST_F(ConnectionFactoryImplTest, CanarySucceedsRetryDuringLogin) {
 436   factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
 437   factory()->Connect();
 438   WaitForConnections();
 439   base::TimeTicks initial_backoff = factory()->NextRetryAttempt();
 440   EXPECT_FALSE(initial_backoff.is_null());
 441
 442   factory()->SetDelayLogin(true);
 443   factory()->SetConnectResult(net::OK);
 444   factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_WIFI);
 445   WaitForConnections();
 446   EXPECT_FALSE(factory()->IsEndpointReachable());
 447
 448   // Pump the loop, to ensure the pending backoff retry has no effect.
 449   base::MessageLoop::current()->PostDelayedTask(
 450       FROM_HERE,
 451       base::MessageLoop::QuitClosure(),
 452       base::TimeDelta::FromMilliseconds(1));
 453   WaitForConnections();
 454 }
 455
 456 // Fail after successful connection via signal reset.
 457 TEST_F(ConnectionFactoryImplTest, FailViaSignalReset) {
 458   factory()->SetConnectResult(net::OK);
 459   factory()->Connect();
 460   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
 461
 462   factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
 463   EXPECT_FALSE(factory()->NextRetryAttempt().is_null());
 464   EXPECT_FALSE(factory()->IsEndpointReachable());
 465 }
 466
 467 TEST_F(ConnectionFactoryImplTest, IgnoreResetWhileConnecting) {
 468   factory()->SetConnectResult(net::OK);
 469   factory()->Connect();
 470   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
 471
 472   factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
 473   base::TimeTicks retry_time = factory()->NextRetryAttempt();
 474   EXPECT_FALSE(retry_time.is_null());
 475   EXPECT_FALSE(factory()->IsEndpointReachable());
 476
 477   const int kNumAttempts = 5;
 478   for (int i = 0; i < kNumAttempts; ++i)
 479     factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
 480   EXPECT_EQ(retry_time, factory()->NextRetryAttempt());
 481   EXPECT_FALSE(factory()->IsEndpointReachable());
 482 }
 483
 484 // Go into backoff due to connection failure. On successful connection, receive
 485 // a signal reset. The original backoff should be restored and extended, rather
 486 // than a new backoff starting from scratch.
 487 TEST_F(ConnectionFactoryImplTest, SignalResetRestoresBackoff) {
 488   factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
 489   base::TimeTicks connect_time = factory()->tick_clock()->NowTicks();
 490   factory()->Connect();
 491   WaitForConnections();
 492   base::TimeTicks retry_time = factory()->NextRetryAttempt();
 493   EXPECT_FALSE(retry_time.is_null());
 494
 495   factory()->SetConnectResult(net::OK);
 496   connect_time = factory()->tick_clock()->NowTicks();
 497   WaitForConnections();
 498   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
 499
 500   factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
 501   EXPECT_FALSE(factory()->IsEndpointReachable());
 502   EXPECT_FALSE(connected_server().is_valid());
 503   EXPECT_NE(retry_time, factory()->NextRetryAttempt());
 504   retry_time = factory()->NextRetryAttempt();
 505   EXPECT_FALSE(retry_time.is_null());
 506   EXPECT_GE((retry_time - connect_time).InMilliseconds(),
 507             CalculateBackoff(2));
 508
 509   factory()->SetConnectResult(net::OK);
 510   connect_time = factory()->tick_clock()->NowTicks();
 511   factory()->tick_clock()->Advance(
 512       factory()->NextRetryAttempt() - connect_time);
 513   WaitForConnections();
 514   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
 515   EXPECT_TRUE(factory()->IsEndpointReachable());
 516   EXPECT_TRUE(connected_server().is_valid());
 517
 518   factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
 519   EXPECT_NE(retry_time, factory()->NextRetryAttempt());
 520   retry_time = factory()->NextRetryAttempt();
 521   EXPECT_FALSE(retry_time.is_null());
 522   EXPECT_GE((retry_time - connect_time).InMilliseconds(),
 523             CalculateBackoff(3));
 524   EXPECT_FALSE(factory()->IsEndpointReachable());
 525   EXPECT_FALSE(connected_server().is_valid());
 526 }
 527
 528 // When the network is disconnected, close the socket and suppress further
 529 // connection attempts until the network returns.
 530 // Disabled while crbug.com/396687 is being investigated.
 531 TEST_F(ConnectionFactoryImplTest, DISABLED_SuppressConnectWhenNoNetwork) {
 532   factory()->SetConnectResult(net::OK);
 533   factory()->Connect();
 534   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
 535   EXPECT_TRUE(factory()->IsEndpointReachable());
 536
 537   // Advance clock so the login window reset isn't encountered.
 538   factory()->tick_clock()->Advance(base::TimeDelta::FromSeconds(11));
 539
 540   // Will trigger reset, but will not attempt a new connection.
 541   factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_NONE);
 542   EXPECT_FALSE(factory()->IsEndpointReachable());
 543   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
 544
 545   // When the network returns, attempt to connect.
 546   factory()->SetConnectResult(net::OK);
 547   factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_4G);
 548   WaitForConnections();
 549
 550   EXPECT_TRUE(factory()->IsEndpointReachable());
 551   EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
 552 }
 553
 554 }  // namespace gcm