sync/engine/sync_scheduler_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/bind.h"
   6 #include "base/callback.h"
   7 #include "base/compiler_specific.h"
   8 #include "base/memory/weak_ptr.h"
   9 #include "base/message_loop/message_loop.h"
  10 #include "base/test/test_timeouts.h"
  11 #include "sync/engine/backoff_delay_provider.h"
  12 #include "sync/engine/sync_scheduler_impl.h"
  13 #include "sync/engine/syncer.h"
  14 #include "sync/internal_api/public/base/cancelation_signal.h"
  15 #include "sync/internal_api/public/base/model_type_test_util.h"
  16 #include "sync/sessions/test_util.h"
  17 #include "sync/test/callback_counter.h"
  18 #include "sync/test/engine/fake_model_worker.h"
  19 #include "sync/test/engine/mock_connection_manager.h"
  20 #include "sync/test/engine/mock_nudge_handler.h"
  21 #include "sync/test/engine/test_directory_setter_upper.h"
  22 #include "sync/test/mock_invalidation.h"
  23 #include "sync/util/extensions_activity.h"
  24 #include "testing/gmock/include/gmock/gmock.h"
  25 #include "testing/gtest/include/gtest/gtest.h"
  26
  27 using base::TimeDelta;
  28 using base::TimeTicks;
  29 using testing::_;
  30 using testing::AtLeast;
  31 using testing::DoAll;
  32 using testing::Invoke;
  33 using testing::Mock;
  34 using testing::Return;
  35 using testing::WithArg;
  36 using testing::WithArgs;
  37 using testing::WithoutArgs;
  38
  39 namespace syncer {
  40 using sessions::SyncSession;
  41 using sessions::SyncSessionContext;
  42 using sync_pb::GetUpdatesCallerInfo;
  43
  44 class MockSyncer : public Syncer {
  45  public:
  46   MockSyncer();
  47   MOCK_METHOD3(NormalSyncShare, bool(ModelTypeSet,
  48                                      const sessions::NudgeTracker&,
  49                                      sessions::SyncSession*));
  50   MOCK_METHOD3(ConfigureSyncShare,
  51                bool(ModelTypeSet,
  52                     sync_pb::GetUpdatesCallerInfo::GetUpdatesSource,
  53                     SyncSession*));
  54   MOCK_METHOD2(PollSyncShare, bool(ModelTypeSet, sessions::SyncSession*));
  55 };
  56
  57 MockSyncer::MockSyncer()
  58   : Syncer(NULL) {}
  59
  60 typedef std::vector<TimeTicks> SyncShareTimes;
  61
  62 void QuitLoopNow() {
  63   // We use QuitNow() instead of Quit() as the latter may get stalled
  64   // indefinitely in the presence of repeated timers with low delays
  65   // and a slow test (e.g., ThrottlingDoesThrottle [which has a poll
  66   // delay of 5ms] run under TSAN on the trybots).
  67   base::MessageLoop::current()->QuitNow();
  68 }
  69
  70 void RunLoop() {
  71   base::MessageLoop::current()->Run();
  72 }
  73
  74 void PumpLoop() {
  75   // Do it this way instead of RunAllPending to pump loop exactly once
  76   // (necessary in the presence of timers; see comment in
  77   // QuitLoopNow).
  78   base::MessageLoop::current()->PostTask(FROM_HERE, base::Bind(&QuitLoopNow));
  79   RunLoop();
  80 }
  81
  82 void PumpLoopFor(base::TimeDelta time) {
  83   // Allow the loop to run for the specified amount of time.
  84   base::MessageLoop::current()->PostDelayedTask(
  85       FROM_HERE, base::Bind(&QuitLoopNow), time);
  86   RunLoop();
  87 }
  88
  89 ModelSafeRoutingInfo TypesToRoutingInfo(ModelTypeSet types) {
  90   ModelSafeRoutingInfo routes;
  91   for (ModelTypeSet::Iterator iter = types.First(); iter.Good(); iter.Inc()) {
  92     routes[iter.Get()] = GROUP_PASSIVE;
  93   }
  94   return routes;
  95 }
  96
  97
  98 static const size_t kMinNumSamples = 5;
  99
 100 // Test harness for the SyncScheduler.  Test the delays and backoff timers used
 101 // in response to various events.
 102 //
 103 // These tests execute in real time with real timers.  We try to keep the
 104 // delays short, but there is a limit to how short we can make them.  The
 105 // timers on some platforms (ie. Windows) have a timer resolution greater than
 106 // 1ms.  Using 1ms delays may result in test flakiness.
 107 //
 108 // See crbug.com/402212 for more info.
 109 class SyncSchedulerTest : public testing::Test {
 110  public:
 111   SyncSchedulerTest() : syncer_(NULL), delay_(NULL), weak_ptr_factory_(this) {}
 112
 113   class MockDelayProvider : public BackoffDelayProvider {
 114    public:
 115     MockDelayProvider() : BackoffDelayProvider(
 116         TimeDelta::FromSeconds(kInitialBackoffRetrySeconds),
 117         TimeDelta::FromSeconds(kInitialBackoffImmediateRetrySeconds)) {
 118     }
 119
 120     MOCK_METHOD1(GetDelay, TimeDelta(const TimeDelta&));
 121   };
 122
 123   virtual void SetUp() {
 124     dir_maker_.SetUp();
 125     syncer_ = new testing::StrictMock<MockSyncer>();
 126     delay_ = NULL;
 127     extensions_activity_ = new ExtensionsActivity();
 128
 129     routing_info_[THEMES] = GROUP_UI;
 130     routing_info_[TYPED_URLS] = GROUP_DB;
 131     routing_info_[THEMES] = GROUP_UI;
 132     routing_info_[NIGORI] = GROUP_PASSIVE;
 133
 134     workers_.clear();
 135     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_UI)));
 136     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_DB)));
 137     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_PASSIVE)));
 138
 139     connection_.reset(new MockConnectionManager(directory(),
 140                                                 &cancelation_signal_));
 141     connection_->SetServerReachable();
 142
 143     model_type_registry_.reset(
 144         new ModelTypeRegistry(workers_, directory(), &mock_nudge_handler_));
 145
 146     context_.reset(new SyncSessionContext(
 147             connection_.get(), directory(),
 148             extensions_activity_.get(),
 149             std::vector<SyncEngineEventListener*>(), NULL,
 150             model_type_registry_.get(),
 151             true,  // enable keystore encryption
 152             false,  // force enable pre-commit GU avoidance
 153             "fake_invalidator_client_id"));
 154     context_->SetRoutingInfo(routing_info_);
 155     context_->set_notifications_enabled(true);
 156     context_->set_account_name("Test");
 157     scheduler_.reset(
 158         new SyncSchedulerImpl("TestSyncScheduler",
 159             BackoffDelayProvider::FromDefaults(),
 160             context(),
 161             syncer_));
 162     scheduler_->SetDefaultNudgeDelay(default_delay());
 163   }
 164
 165   SyncSchedulerImpl* scheduler() { return scheduler_.get(); }
 166   const ModelSafeRoutingInfo& routing_info() { return routing_info_; }
 167   MockSyncer* syncer() { return syncer_; }
 168   MockDelayProvider* delay() { return delay_; }
 169   MockConnectionManager* connection() { return connection_.get(); }
 170   TimeDelta default_delay() { return TimeDelta::FromSeconds(0); }
 171   TimeDelta timeout() {
 172     return TestTimeouts::action_timeout();
 173   }
 174
 175   virtual void TearDown() {
 176     PumpLoop();
 177     scheduler_.reset();
 178     PumpLoop();
 179     dir_maker_.TearDown();
 180   }
 181
 182   void AnalyzePollRun(const SyncShareTimes& times, size_t min_num_samples,
 183       const TimeTicks& optimal_start, const TimeDelta& poll_interval) {
 184     EXPECT_GE(times.size(), min_num_samples);
 185     for (size_t i = 0; i < times.size(); i++) {
 186       SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
 187       TimeTicks optimal_next_sync = optimal_start + poll_interval * i;
 188       EXPECT_GE(times[i], optimal_next_sync);
 189     }
 190   }
 191
 192   void DoQuitLoopNow() {
 193     QuitLoopNow();
 194   }
 195
 196   void StartSyncScheduler(SyncScheduler::Mode mode) {
 197     scheduler()->Start(mode);
 198   }
 199
 200   // This stops the scheduler synchronously.
 201   void StopSyncScheduler() {
 202     base::MessageLoop::current()->PostTask(
 203         FROM_HERE,
 204         base::Bind(&SyncSchedulerTest::DoQuitLoopNow,
 205                    weak_ptr_factory_.GetWeakPtr()));
 206     RunLoop();
 207   }
 208
 209   bool RunAndGetBackoff() {
 210     ModelTypeSet nudge_types(THEMES);
 211     StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 212
 213     scheduler()->ScheduleLocalNudge(nudge_types, FROM_HERE);
 214     RunLoop();
 215
 216     return scheduler()->IsBackingOff();
 217   }
 218
 219   void UseMockDelayProvider() {
 220     delay_ = new MockDelayProvider();
 221     scheduler_->delay_provider_.reset(delay_);
 222   }
 223
 224   SyncSessionContext* context() { return context_.get(); }
 225
 226   ModelTypeSet GetThrottledTypes() {
 227     return scheduler_->nudge_tracker_.GetThrottledTypes();
 228   }
 229
 230   base::TimeDelta GetRetryTimerDelay() {
 231     EXPECT_TRUE(scheduler_->retry_timer_.IsRunning());
 232     return scheduler_->retry_timer_.GetCurrentDelay();
 233   }
 234
 235   static scoped_ptr<InvalidationInterface> BuildInvalidation(
 236       int64 version,
 237       const std::string& payload) {
 238     return MockInvalidation::Build(version, payload)
 239         .PassAs<InvalidationInterface>();
 240   }
 241
 242  private:
 243   syncable::Directory* directory() {
 244     return dir_maker_.directory();
 245   }
 246
 247   base::MessageLoop loop_;
 248   TestDirectorySetterUpper dir_maker_;
 249   CancelationSignal cancelation_signal_;
 250   scoped_ptr<MockConnectionManager> connection_;
 251   scoped_ptr<ModelTypeRegistry> model_type_registry_;
 252   scoped_ptr<SyncSessionContext> context_;
 253   scoped_ptr<SyncSchedulerImpl> scheduler_;
 254   MockNudgeHandler mock_nudge_handler_;
 255   MockSyncer* syncer_;
 256   MockDelayProvider* delay_;
 257   std::vector<scoped_refptr<ModelSafeWorker> > workers_;
 258   scoped_refptr<ExtensionsActivity> extensions_activity_;
 259   ModelSafeRoutingInfo routing_info_;
 260   base::WeakPtrFactory<SyncSchedulerTest> weak_ptr_factory_;
 261 };
 262
 263 void RecordSyncShareImpl(SyncShareTimes* times) {
 264   times->push_back(TimeTicks::Now());
 265 }
 266
 267 ACTION_P(RecordSyncShare, times) {
 268   RecordSyncShareImpl(times);
 269   if (base::MessageLoop::current()->is_running())
 270     QuitLoopNow();
 271   return true;
 272 }
 273
 274 ACTION_P2(RecordSyncShareMultiple, times, quit_after) {
 275   RecordSyncShareImpl(times);
 276   EXPECT_LE(times->size(), quit_after);
 277   if (times->size() >= quit_after &&
 278       base::MessageLoop::current()->is_running()) {
 279     QuitLoopNow();
 280   }
 281   return true;
 282 }
 283
 284 ACTION_P(StopScheduler, scheduler) {
 285   scheduler->Stop();
 286 }
 287
 288 ACTION(AddFailureAndQuitLoopNow) {
 289   ADD_FAILURE();
 290   QuitLoopNow();
 291   return true;
 292 }
 293
 294 ACTION(QuitLoopNowAction) {
 295   QuitLoopNow();
 296   return true;
 297 }
 298
 299 // Test nudge scheduling.
 300 TEST_F(SyncSchedulerTest, Nudge) {
 301   SyncShareTimes times;
 302   ModelTypeSet model_types(THEMES);
 303
 304   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 305       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 306                       RecordSyncShare(&times)))
 307       .RetiresOnSaturation();
 308
 309   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 310
 311   scheduler()->ScheduleLocalNudge(model_types, FROM_HERE);
 312   RunLoop();
 313
 314   Mock::VerifyAndClearExpectations(syncer());
 315
 316   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 317   SyncShareTimes times2;
 318   model_types.Remove(THEMES);
 319   model_types.Put(TYPED_URLS);
 320   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 321       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 322                       RecordSyncShare(&times2)));
 323   scheduler()->ScheduleLocalNudge(model_types, FROM_HERE);
 324   RunLoop();
 325 }
 326
 327 // Make sure a regular config command is scheduled fine in the absence of any
 328 // errors.
 329 TEST_F(SyncSchedulerTest, Config) {
 330   SyncShareTimes times;
 331   const ModelTypeSet model_types(THEMES);
 332
 333   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 334       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 335                       RecordSyncShare(&times)));
 336
 337   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 338
 339   CallbackCounter ready_counter;
 340   CallbackCounter retry_counter;
 341   ConfigurationParams params(
 342       GetUpdatesCallerInfo::RECONFIGURATION,
 343       model_types,
 344       TypesToRoutingInfo(model_types),
 345       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 346       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 347   scheduler()->ScheduleConfiguration(params);
 348   PumpLoop();
 349   ASSERT_EQ(1, ready_counter.times_called());
 350   ASSERT_EQ(0, retry_counter.times_called());
 351 }
 352
 353 // Simulate a failure and make sure the config request is retried.
 354 TEST_F(SyncSchedulerTest, ConfigWithBackingOff) {
 355   UseMockDelayProvider();
 356   EXPECT_CALL(*delay(), GetDelay(_))
 357       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(20)));
 358   SyncShareTimes times;
 359   const ModelTypeSet model_types(THEMES);
 360
 361   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 362
 363   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 364       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 365                       RecordSyncShare(&times)))
 366       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 367                       RecordSyncShare(&times)));
 368
 369   CallbackCounter ready_counter;
 370   CallbackCounter retry_counter;
 371   ConfigurationParams params(
 372       GetUpdatesCallerInfo::RECONFIGURATION,
 373       model_types,
 374       TypesToRoutingInfo(model_types),
 375       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 376       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 377   scheduler()->ScheduleConfiguration(params);
 378   RunLoop();
 379   ASSERT_EQ(0, ready_counter.times_called());
 380   ASSERT_EQ(1, retry_counter.times_called());
 381
 382   // RunLoop() will trigger TryCanaryJob which will retry configuration.
 383   // Since retry_task was already called it shouldn't be called again.
 384   RunLoop();
 385   ASSERT_EQ(0, ready_counter.times_called());
 386   ASSERT_EQ(1, retry_counter.times_called());
 387
 388   Mock::VerifyAndClearExpectations(syncer());
 389
 390   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 391       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 392                       RecordSyncShare(&times)));
 393   RunLoop();
 394
 395   ASSERT_EQ(1, ready_counter.times_called());
 396 }
 397
 398 // Simuilate SyncSchedulerImpl::Stop being called in the middle of Configure.
 399 // This can happen if server returns NOT_MY_BIRTHDAY.
 400 TEST_F(SyncSchedulerTest, ConfigWithStop) {
 401   UseMockDelayProvider();
 402   EXPECT_CALL(*delay(), GetDelay(_))
 403       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(20)));
 404   SyncShareTimes times;
 405   const ModelTypeSet model_types(THEMES);
 406
 407   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 408
 409   // Make ConfigureSyncShare call scheduler->Stop(). It is not supposed to call
 410   // retry_task or dereference configuration params.
 411   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 412       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 413                       StopScheduler(scheduler()),
 414                       RecordSyncShare(&times)));
 415
 416   CallbackCounter ready_counter;
 417   CallbackCounter retry_counter;
 418   ConfigurationParams params(
 419       GetUpdatesCallerInfo::RECONFIGURATION,
 420       model_types,
 421       TypesToRoutingInfo(model_types),
 422       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 423       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 424   scheduler()->ScheduleConfiguration(params);
 425   PumpLoop();
 426   ASSERT_EQ(0, ready_counter.times_called());
 427   ASSERT_EQ(0, retry_counter.times_called());
 428 }
 429
 430 // Issue a nudge when the config has failed. Make sure both the config and
 431 // nudge are executed.
 432 TEST_F(SyncSchedulerTest, NudgeWithConfigWithBackingOff) {
 433   const ModelTypeSet model_types(THEMES);
 434   UseMockDelayProvider();
 435   EXPECT_CALL(*delay(), GetDelay(_))
 436       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(50)));
 437   SyncShareTimes times;
 438
 439   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 440
 441   // Request a configure and make sure it fails.
 442   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 443       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 444                       RecordSyncShare(&times)));
 445   CallbackCounter ready_counter;
 446   CallbackCounter retry_counter;
 447   ConfigurationParams params(
 448       GetUpdatesCallerInfo::RECONFIGURATION,
 449       model_types,
 450       TypesToRoutingInfo(model_types),
 451       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 452       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 453   scheduler()->ScheduleConfiguration(params);
 454   RunLoop();
 455   ASSERT_EQ(0, ready_counter.times_called());
 456   ASSERT_EQ(1, retry_counter.times_called());
 457   Mock::VerifyAndClearExpectations(syncer());
 458
 459   // Ask for a nudge while dealing with repeated configure failure.
 460   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 461       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 462                       RecordSyncShare(&times)));
 463   scheduler()->ScheduleLocalNudge(model_types, FROM_HERE);
 464   RunLoop();
 465   // Note that we're not RunLoop()ing for the NUDGE we just scheduled, but
 466   // for the first retry attempt from the config job (after
 467   // waiting ~+/- 50ms).
 468   Mock::VerifyAndClearExpectations(syncer());
 469   ASSERT_EQ(0, ready_counter.times_called());
 470
 471   // Let the next configure retry succeed.
 472   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 473       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 474                       RecordSyncShare(&times)));
 475   RunLoop();
 476
 477   // Now change the mode so nudge can execute.
 478   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 479       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 480                       RecordSyncShare(&times)));
 481   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 482   PumpLoop();
 483 }
 484
 485 // Test that nudges are coalesced.
 486 TEST_F(SyncSchedulerTest, NudgeCoalescing) {
 487   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 488
 489   SyncShareTimes times;
 490   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 491       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 492                       RecordSyncShare(&times)));
 493   const ModelTypeSet types1(THEMES), types2(TYPED_URLS), types3(THEMES);
 494   TimeTicks optimal_time = TimeTicks::Now() + default_delay();
 495   scheduler()->ScheduleLocalNudge(types1, FROM_HERE);
 496   scheduler()->ScheduleLocalNudge(types2, FROM_HERE);
 497   RunLoop();
 498
 499   ASSERT_EQ(1U, times.size());
 500   EXPECT_GE(times[0], optimal_time);
 501
 502   Mock::VerifyAndClearExpectations(syncer());
 503
 504   SyncShareTimes times2;
 505   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 506       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 507                       RecordSyncShare(&times2)));
 508   scheduler()->ScheduleLocalNudge(types3, FROM_HERE);
 509   RunLoop();
 510 }
 511
 512 // Test that nudges are coalesced.
 513 TEST_F(SyncSchedulerTest, NudgeCoalescingWithDifferentTimings) {
 514   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 515
 516   SyncShareTimes times;
 517   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 518       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 519                       RecordSyncShare(&times)));
 520   ModelTypeSet types1(THEMES), types2(TYPED_URLS), types3;
 521
 522   // Create a huge time delay.
 523   TimeDelta delay = TimeDelta::FromDays(1);
 524
 525   std::map<ModelType, TimeDelta> delay_map;
 526   delay_map[types1.First().Get()] = delay;
 527   scheduler()->OnReceivedCustomNudgeDelays(delay_map);
 528   scheduler()->ScheduleLocalNudge(types1, FROM_HERE);
 529   scheduler()->ScheduleLocalNudge(types2, FROM_HERE);
 530
 531   TimeTicks min_time = TimeTicks::Now();
 532   TimeTicks max_time = TimeTicks::Now() + delay;
 533
 534   RunLoop();
 535   Mock::VerifyAndClearExpectations(syncer());
 536
 537   // Make sure the sync happened at the right time.
 538   ASSERT_EQ(1U, times.size());
 539   EXPECT_GE(times[0], min_time);
 540   EXPECT_LE(times[0], max_time);
 541 }
 542
 543 // Test nudge scheduling.
 544 TEST_F(SyncSchedulerTest, NudgeWithStates) {
 545   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 546
 547   SyncShareTimes times1;
 548   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 549       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 550                       RecordSyncShare(&times1)))
 551       .RetiresOnSaturation();
 552   scheduler()->ScheduleInvalidationNudge(
 553       THEMES, BuildInvalidation(10, "test"), FROM_HERE);
 554   RunLoop();
 555
 556   Mock::VerifyAndClearExpectations(syncer());
 557
 558   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 559   SyncShareTimes times2;
 560   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 561       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 562                       RecordSyncShare(&times2)));
 563   scheduler()->ScheduleInvalidationNudge(
 564       TYPED_URLS, BuildInvalidation(10, "test2"), FROM_HERE);
 565   RunLoop();
 566 }
 567
 568 // Test that polling works as expected.
 569 TEST_F(SyncSchedulerTest, Polling) {
 570   SyncShareTimes times;
 571   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 572   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 573       .WillRepeatedly(
 574           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 575                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 576
 577   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
 578
 579   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 580   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 581
 582   // Run again to wait for polling.
 583   RunLoop();
 584
 585   StopSyncScheduler();
 586   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 587 }
 588
 589 // Test that the short poll interval is used.
 590 TEST_F(SyncSchedulerTest, PollNotificationsDisabled) {
 591   SyncShareTimes times;
 592   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 593   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 594       .WillRepeatedly(
 595           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 596                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 597
 598   scheduler()->OnReceivedShortPollIntervalUpdate(poll_interval);
 599   scheduler()->SetNotificationsEnabled(false);
 600
 601   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 602   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 603
 604   // Run again to wait for polling.
 605   RunLoop();
 606
 607   StopSyncScheduler();
 608   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 609 }
 610
 611 // Test that polling intervals are updated when needed.
 612 TEST_F(SyncSchedulerTest, PollIntervalUpdate) {
 613   SyncShareTimes times;
 614   TimeDelta poll1(TimeDelta::FromMilliseconds(120));
 615   TimeDelta poll2(TimeDelta::FromMilliseconds(30));
 616   scheduler()->OnReceivedLongPollIntervalUpdate(poll1);
 617   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 618       .WillOnce(DoAll(
 619           WithArgs<0,1>(
 620               sessions::test_util::SimulatePollIntervalUpdate(poll2)),
 621           Return(true)))
 622       .WillRepeatedly(
 623           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 624                 WithArg<1>(
 625                     RecordSyncShareMultiple(&times, kMinNumSamples))));
 626
 627   TimeTicks optimal_start = TimeTicks::Now() + poll1 + poll2;
 628   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 629
 630   // Run again to wait for polling.
 631   RunLoop();
 632
 633   StopSyncScheduler();
 634   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll2);
 635 }
 636
 637 // Test that no syncing occurs when throttled.
 638 TEST_F(SyncSchedulerTest, ThrottlingDoesThrottle) {
 639   const ModelTypeSet types(THEMES);
 640   TimeDelta poll(TimeDelta::FromMilliseconds(20));
 641   TimeDelta throttle(TimeDelta::FromMinutes(10));
 642   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 643
 644   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 645       .WillOnce(DoAll(
 646           WithArg<2>(sessions::test_util::SimulateThrottled(throttle)),
 647           Return(true)))
 648       .WillRepeatedly(AddFailureAndQuitLoopNow());
 649
 650   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 651
 652   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
 653   PumpLoop();
 654
 655   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 656
 657   CallbackCounter ready_counter;
 658   CallbackCounter retry_counter;
 659   ConfigurationParams params(
 660       GetUpdatesCallerInfo::RECONFIGURATION,
 661       types,
 662       TypesToRoutingInfo(types),
 663       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 664       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 665   scheduler()->ScheduleConfiguration(params);
 666   PumpLoop();
 667   ASSERT_EQ(0, ready_counter.times_called());
 668   ASSERT_EQ(1, retry_counter.times_called());
 669
 670 }
 671
 672 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromPoll) {
 673   SyncShareTimes times;
 674   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 675   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 676   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 677
 678   ::testing::InSequence seq;
 679   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 680       .WillOnce(DoAll(
 681           WithArg<1>(sessions::test_util::SimulateThrottled(throttle1)),
 682           Return(true)))
 683       .RetiresOnSaturation();
 684   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 685       .WillRepeatedly(
 686           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 687                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 688
 689   TimeTicks optimal_start = TimeTicks::Now() + poll + throttle1;
 690   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 691
 692   // Run again to wait for polling.
 693   RunLoop();
 694
 695   StopSyncScheduler();
 696   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll);
 697 }
 698
 699 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromNudge) {
 700   SyncShareTimes times;
 701   TimeDelta poll(TimeDelta::FromDays(1));
 702   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 703   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 704
 705   ::testing::InSequence seq;
 706   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 707       .WillOnce(DoAll(
 708           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 709           Return(true)))
 710       .RetiresOnSaturation();
 711   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 712       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 713                       QuitLoopNowAction()));
 714
 715   const ModelTypeSet types(THEMES);
 716   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 717   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
 718
 719   PumpLoop(); // To get PerformDelayedNudge called.
 720   PumpLoop(); // To get TrySyncSessionJob called
 721   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 722   RunLoop();
 723   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 724
 725   StopSyncScheduler();
 726 }
 727
 728 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromConfigure) {
 729   SyncShareTimes times;
 730   TimeDelta poll(TimeDelta::FromDays(1));
 731   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 732   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 733
 734   ::testing::InSequence seq;
 735   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 736       .WillOnce(DoAll(
 737           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 738           Return(true)))
 739       .RetiresOnSaturation();
 740   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 741       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 742                       QuitLoopNowAction()));
 743
 744   const ModelTypeSet types(THEMES);
 745   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 746
 747   CallbackCounter ready_counter;
 748   CallbackCounter retry_counter;
 749   ConfigurationParams params(
 750       GetUpdatesCallerInfo::RECONFIGURATION,
 751       types,
 752       TypesToRoutingInfo(types),
 753       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 754       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 755   scheduler()->ScheduleConfiguration(params);
 756   PumpLoop();
 757   EXPECT_EQ(0, ready_counter.times_called());
 758   EXPECT_EQ(1, retry_counter.times_called());
 759   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 760
 761   RunLoop();
 762   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 763
 764   StopSyncScheduler();
 765 }
 766
 767 TEST_F(SyncSchedulerTest, TypeThrottlingBlocksNudge) {
 768   UseMockDelayProvider();
 769   EXPECT_CALL(*delay(), GetDelay(_))
 770       .WillRepeatedly(Return(default_delay()));
 771
 772   TimeDelta poll(TimeDelta::FromDays(1));
 773   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 774   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 775
 776   const ModelTypeSet types(THEMES);
 777
 778   ::testing::InSequence seq;
 779   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 780       .WillOnce(DoAll(
 781           WithArg<2>(
 782               sessions::test_util::SimulateTypesThrottled(types, throttle1)),
 783           Return(true)))
 784       .RetiresOnSaturation();
 785
 786   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 787   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
 788   PumpLoop(); // To get PerformDelayedNudge called.
 789   PumpLoop(); // To get TrySyncSessionJob called
 790   EXPECT_TRUE(GetThrottledTypes().HasAll(types));
 791
 792   // This won't cause a sync cycle because the types are throttled.
 793   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
 794   PumpLoop();
 795
 796   StopSyncScheduler();
 797 }
 798
 799 TEST_F(SyncSchedulerTest, TypeThrottlingDoesBlockOtherSources) {
 800   UseMockDelayProvider();
 801   EXPECT_CALL(*delay(), GetDelay(_))
 802       .WillRepeatedly(Return(default_delay()));
 803
 804   SyncShareTimes times;
 805   TimeDelta poll(TimeDelta::FromDays(1));
 806   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 807   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 808
 809   const ModelTypeSet throttled_types(THEMES);
 810   const ModelTypeSet unthrottled_types(PREFERENCES);
 811
 812   ::testing::InSequence seq;
 813   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 814       .WillOnce(DoAll(
 815           WithArg<2>(
 816               sessions::test_util::SimulateTypesThrottled(
 817                   throttled_types, throttle1)),
 818           Return(true)))
 819       .RetiresOnSaturation();
 820
 821   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 822   scheduler()->ScheduleLocalNudge(throttled_types, FROM_HERE);
 823   PumpLoop(); // To get PerformDelayedNudge called.
 824   PumpLoop(); // To get TrySyncSessionJob called
 825   EXPECT_TRUE(GetThrottledTypes().HasAll(throttled_types));
 826
 827   // Ignore invalidations for throttled types.
 828   scheduler()->ScheduleInvalidationNudge(
 829       THEMES, BuildInvalidation(10, "test"), FROM_HERE);
 830   PumpLoop();
 831
 832   // Ignore refresh requests for throttled types.
 833   scheduler()->ScheduleLocalRefreshRequest(throttled_types, FROM_HERE);
 834   PumpLoop();
 835
 836   Mock::VerifyAndClearExpectations(syncer());
 837
 838   // Local nudges for non-throttled types will trigger a sync.
 839   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 840       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 841                             RecordSyncShare(&times)));
 842   scheduler()->ScheduleLocalNudge(unthrottled_types, FROM_HERE);
 843   RunLoop();
 844   Mock::VerifyAndClearExpectations(syncer());
 845
 846   StopSyncScheduler();
 847 }
 848
 849 // Test nudges / polls don't run in config mode and config tasks do.
 850 TEST_F(SyncSchedulerTest, ConfigurationMode) {
 851   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 852   SyncShareTimes times;
 853   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 854
 855   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 856
 857   const ModelTypeSet nudge_types(TYPED_URLS);
 858   scheduler()->ScheduleLocalNudge(nudge_types, FROM_HERE);
 859   scheduler()->ScheduleLocalNudge(nudge_types, FROM_HERE);
 860
 861   const ModelTypeSet config_types(THEMES);
 862
 863   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 864       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 865                       RecordSyncShare(&times)))
 866       .RetiresOnSaturation();
 867   CallbackCounter ready_counter;
 868   CallbackCounter retry_counter;
 869   ConfigurationParams params(
 870       GetUpdatesCallerInfo::RECONFIGURATION,
 871       config_types,
 872       TypesToRoutingInfo(config_types),
 873       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 874       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 875   scheduler()->ScheduleConfiguration(params);
 876   RunLoop();
 877   ASSERT_EQ(1, ready_counter.times_called());
 878   ASSERT_EQ(0, retry_counter.times_called());
 879
 880   Mock::VerifyAndClearExpectations(syncer());
 881
 882   // Switch to NORMAL_MODE to ensure NUDGES were properly saved and run.
 883   scheduler()->OnReceivedLongPollIntervalUpdate(TimeDelta::FromDays(1));
 884   SyncShareTimes times2;
 885   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 886       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 887                       RecordSyncShare(&times2)));
 888
 889   // TODO(tim): Figure out how to remove this dangerous need to reset
 890   // routing info between mode switches.
 891   context()->SetRoutingInfo(routing_info());
 892   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 893
 894   RunLoop();
 895   Mock::VerifyAndClearExpectations(syncer());
 896 }
 897
 898 class BackoffTriggersSyncSchedulerTest : public SyncSchedulerTest {
 899   virtual void SetUp() {
 900     SyncSchedulerTest::SetUp();
 901     UseMockDelayProvider();
 902     EXPECT_CALL(*delay(), GetDelay(_))
 903         .WillRepeatedly(Return(TimeDelta::FromMilliseconds(10)));
 904   }
 905
 906   virtual void TearDown() {
 907     StopSyncScheduler();
 908     SyncSchedulerTest::TearDown();
 909   }
 910 };
 911
 912 // Have the sycner fail during commit.  Expect that the scheduler enters
 913 // backoff.
 914 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnce) {
 915   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 916       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
 917                       QuitLoopNowAction()));
 918   EXPECT_TRUE(RunAndGetBackoff());
 919 }
 920
 921 // Have the syncer fail during download updates and succeed on the first
 922 // retry.  Expect that this clears the backoff state.
 923 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadOnceThenSucceed) {
 924   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 925       .WillOnce(DoAll(
 926           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 927           Return(true)))
 928       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 929                       QuitLoopNowAction()));
 930   EXPECT_FALSE(RunAndGetBackoff());
 931 }
 932
 933 // Have the syncer fail during commit and succeed on the first retry.  Expect
 934 // that this clears the backoff state.
 935 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnceThenSucceed) {
 936   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 937       .WillOnce(DoAll(
 938           Invoke(sessions::test_util::SimulateCommitFailed),
 939           Return(true)))
 940       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 941                       QuitLoopNowAction()));
 942   EXPECT_FALSE(RunAndGetBackoff());
 943 }
 944
 945 // Have the syncer fail to download updates and fail again on the retry.
 946 // Expect this will leave the scheduler in backoff.
 947 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadTwice) {
 948   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 949       .WillOnce(DoAll(
 950           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 951           Return(true)))
 952       .WillRepeatedly(DoAll(
 953               Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 954               QuitLoopNowAction()));
 955   EXPECT_TRUE(RunAndGetBackoff());
 956 }
 957
 958 // Have the syncer fail to get the encryption key yet succeed in downloading
 959 // updates. Expect this will leave the scheduler in backoff.
 960 TEST_F(BackoffTriggersSyncSchedulerTest, FailGetEncryptionKey) {
 961   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 962       .WillOnce(DoAll(
 963           Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
 964           Return(true)))
 965       .WillRepeatedly(DoAll(
 966               Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
 967               QuitLoopNowAction()));
 968   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 969
 970   ModelTypeSet types(THEMES);
 971   CallbackCounter ready_counter;
 972   CallbackCounter retry_counter;
 973   ConfigurationParams params(
 974       GetUpdatesCallerInfo::RECONFIGURATION,
 975       types,
 976       TypesToRoutingInfo(types),
 977       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 978       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 979   scheduler()->ScheduleConfiguration(params);
 980   RunLoop();
 981
 982   EXPECT_TRUE(scheduler()->IsBackingOff());
 983 }
 984
 985 // Test that no polls or extraneous nudges occur when in backoff.
 986 TEST_F(SyncSchedulerTest, BackoffDropsJobs) {
 987   SyncShareTimes times;
 988   TimeDelta poll(TimeDelta::FromMilliseconds(10));
 989   const ModelTypeSet types(THEMES);
 990   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 991   UseMockDelayProvider();
 992
 993   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 994       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
 995                       RecordSyncShareMultiple(&times, 1U)));
 996   EXPECT_CALL(*delay(), GetDelay(_)).
 997       WillRepeatedly(Return(TimeDelta::FromDays(1)));
 998
 999   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1000
1001   // This nudge should fail and put us into backoff.  Thanks to our mock
1002   // GetDelay() setup above, this will be a long backoff.
1003   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
1004   RunLoop();
1005
1006   // From this point forward, no SyncShare functions should be invoked.
1007   Mock::VerifyAndClearExpectations(syncer());
1008
1009   // Wait a while (10x poll interval) so a few poll jobs will be attempted.
1010   PumpLoopFor(poll * 10);
1011
1012   // Try (and fail) to schedule a nudge.
1013   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
1014
1015   Mock::VerifyAndClearExpectations(syncer());
1016   Mock::VerifyAndClearExpectations(delay());
1017
1018   EXPECT_CALL(*delay(), GetDelay(_)).Times(0);
1019
1020   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1021
1022   CallbackCounter ready_counter;
1023   CallbackCounter retry_counter;
1024   ConfigurationParams params(
1025       GetUpdatesCallerInfo::RECONFIGURATION,
1026       types,
1027       TypesToRoutingInfo(types),
1028       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1029       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1030   scheduler()->ScheduleConfiguration(params);
1031   PumpLoop();
1032   ASSERT_EQ(0, ready_counter.times_called());
1033   ASSERT_EQ(1, retry_counter.times_called());
1034
1035 }
1036
1037 // Test that backoff is shaping traffic properly with consecutive errors.
1038 TEST_F(SyncSchedulerTest, BackoffElevation) {
1039   SyncShareTimes times;
1040   UseMockDelayProvider();
1041
1042   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_)).Times(kMinNumSamples)
1043       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1044           RecordSyncShareMultiple(&times, kMinNumSamples)));
1045
1046   const TimeDelta first = TimeDelta::FromSeconds(kInitialBackoffRetrySeconds);
1047   const TimeDelta second = TimeDelta::FromMilliseconds(20);
1048   const TimeDelta third = TimeDelta::FromMilliseconds(30);
1049   const TimeDelta fourth = TimeDelta::FromMilliseconds(40);
1050   const TimeDelta fifth = TimeDelta::FromMilliseconds(50);
1051   const TimeDelta sixth = TimeDelta::FromDays(1);
1052
1053   EXPECT_CALL(*delay(), GetDelay(first)).WillOnce(Return(second))
1054           .RetiresOnSaturation();
1055   EXPECT_CALL(*delay(), GetDelay(second)).WillOnce(Return(third))
1056           .RetiresOnSaturation();
1057   EXPECT_CALL(*delay(), GetDelay(third)).WillOnce(Return(fourth))
1058           .RetiresOnSaturation();
1059   EXPECT_CALL(*delay(), GetDelay(fourth)).WillOnce(Return(fifth))
1060           .RetiresOnSaturation();
1061   EXPECT_CALL(*delay(), GetDelay(fifth)).WillOnce(Return(sixth));
1062
1063   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1064
1065   // Run again with a nudge.
1066   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1067   RunLoop();
1068
1069   ASSERT_EQ(kMinNumSamples, times.size());
1070   EXPECT_GE(times[1] - times[0], second);
1071   EXPECT_GE(times[2] - times[1], third);
1072   EXPECT_GE(times[3] - times[2], fourth);
1073   EXPECT_GE(times[4] - times[3], fifth);
1074 }
1075
1076 // Test that things go back to normal once a retry makes forward progress.
1077 TEST_F(SyncSchedulerTest, BackoffRelief) {
1078   SyncShareTimes times;
1079   const TimeDelta poll(TimeDelta::FromMilliseconds(10));
1080   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1081   UseMockDelayProvider();
1082
1083   const TimeDelta backoff = TimeDelta::FromMilliseconds(10);
1084   EXPECT_CALL(*delay(), GetDelay(_)).WillOnce(Return(backoff));
1085
1086   // Optimal start for the post-backoff poll party.
1087   TimeTicks optimal_start = TimeTicks::Now();
1088   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1089
1090   // Kick off the test with a failed nudge.
1091   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1092       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1093                       RecordSyncShare(&times)));
1094   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1095   RunLoop();
1096   Mock::VerifyAndClearExpectations(syncer());
1097   TimeTicks optimal_job_time = optimal_start;
1098   ASSERT_EQ(1U, times.size());
1099   EXPECT_GE(times[0], optimal_job_time);
1100
1101   // The retry succeeds.
1102   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1103       .WillOnce(DoAll(
1104               Invoke(sessions::test_util::SimulateNormalSuccess),
1105               RecordSyncShare(&times)));
1106   RunLoop();
1107   Mock::VerifyAndClearExpectations(syncer());
1108   optimal_job_time = optimal_job_time + backoff;
1109   ASSERT_EQ(2U, times.size());
1110   EXPECT_GE(times[1], optimal_job_time);
1111
1112   // Now let the Poll timer do its thing.
1113   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1114       .WillRepeatedly(DoAll(
1115               Invoke(sessions::test_util::SimulatePollSuccess),
1116               RecordSyncShareMultiple(&times, kMinNumSamples)));
1117   RunLoop();
1118   Mock::VerifyAndClearExpectations(syncer());
1119   ASSERT_EQ(kMinNumSamples, times.size());
1120   for (size_t i = 2; i < times.size(); i++) {
1121     optimal_job_time = optimal_job_time + poll;
1122     SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
1123     EXPECT_GE(times[i], optimal_job_time);
1124   }
1125
1126   StopSyncScheduler();
1127 }
1128
1129 // Test that poll failures are ignored.  They should have no effect on
1130 // subsequent poll attempts, nor should they trigger a backoff/retry.
1131 TEST_F(SyncSchedulerTest, TransientPollFailure) {
1132   SyncShareTimes times;
1133   const TimeDelta poll_interval(TimeDelta::FromMilliseconds(10));
1134   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
1135   UseMockDelayProvider(); // Will cause test failure if backoff is initiated.
1136
1137   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1138       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollFailed),
1139                       RecordSyncShare(&times)))
1140       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1141                       RecordSyncShare(&times)));
1142
1143   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1144
1145   // Run the unsucessful poll. The failed poll should not trigger backoff.
1146   RunLoop();
1147   EXPECT_FALSE(scheduler()->IsBackingOff());
1148
1149   // Run the successful poll.
1150   RunLoop();
1151   EXPECT_FALSE(scheduler()->IsBackingOff());
1152 }
1153
1154 // Test that starting the syncer thread without a valid connection doesn't
1155 // break things when a connection is detected.
1156 TEST_F(SyncSchedulerTest, StartWhenNotConnected) {
1157   connection()->SetServerNotReachable();
1158   connection()->UpdateConnectionStatus();
1159   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1160     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1161                     Return(true)))
1162     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1163                     Return(true)));
1164   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1165
1166   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1167   // Should save the nudge for until after the server is reachable.
1168   base::MessageLoop::current()->RunUntilIdle();
1169
1170   scheduler()->OnConnectionStatusChange();
1171   connection()->SetServerReachable();
1172   connection()->UpdateConnectionStatus();
1173   base::MessageLoop::current()->RunUntilIdle();
1174 }
1175
1176 TEST_F(SyncSchedulerTest, ServerConnectionChangeDuringBackoff) {
1177   UseMockDelayProvider();
1178   EXPECT_CALL(*delay(), GetDelay(_))
1179       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1180
1181   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1182   connection()->SetServerNotReachable();
1183   connection()->UpdateConnectionStatus();
1184
1185   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1186     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1187                     Return(true)))
1188     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1189                     Return(true)));
1190
1191   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1192   PumpLoop(); // To get PerformDelayedNudge called.
1193   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1194   ASSERT_TRUE(scheduler()->IsBackingOff());
1195
1196   // Before we run the scheduled canary, trigger a server connection change.
1197   scheduler()->OnConnectionStatusChange();
1198   connection()->SetServerReachable();
1199   connection()->UpdateConnectionStatus();
1200   base::MessageLoop::current()->RunUntilIdle();
1201 }
1202
1203 // This was supposed to test the scenario where we receive a nudge while a
1204 // connection change canary is scheduled, but has not run yet.  Since we've made
1205 // the connection change canary synchronous, this is no longer possible.
1206 TEST_F(SyncSchedulerTest, ConnectionChangeCanaryPreemptedByNudge) {
1207   UseMockDelayProvider();
1208   EXPECT_CALL(*delay(), GetDelay(_))
1209       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1210
1211   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1212   connection()->SetServerNotReachable();
1213   connection()->UpdateConnectionStatus();
1214
1215   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1216     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1217                     Return(true)))
1218     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1219                     Return(true)))
1220     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1221                     QuitLoopNowAction()));
1222
1223   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1224
1225   PumpLoop(); // To get PerformDelayedNudge called.
1226   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1227   ASSERT_TRUE(scheduler()->IsBackingOff());
1228
1229   // Before we run the scheduled canary, trigger a server connection change.
1230   scheduler()->OnConnectionStatusChange();
1231   PumpLoop();
1232   connection()->SetServerReachable();
1233   connection()->UpdateConnectionStatus();
1234   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1235   base::MessageLoop::current()->RunUntilIdle();
1236 }
1237
1238 // Tests that we don't crash trying to run two canaries at once if we receive
1239 // extra connection status change notifications.  See crbug.com/190085.
1240 TEST_F(SyncSchedulerTest, DoubleCanaryInConfigure) {
1241   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
1242       .WillRepeatedly(DoAll(
1243               Invoke(sessions::test_util::SimulateConfigureConnectionFailure),
1244               Return(true)));
1245   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1246   connection()->SetServerNotReachable();
1247   connection()->UpdateConnectionStatus();
1248
1249   ModelTypeSet model_types(THEMES);
1250   CallbackCounter ready_counter;
1251   CallbackCounter retry_counter;
1252   ConfigurationParams params(
1253       GetUpdatesCallerInfo::RECONFIGURATION,
1254       model_types,
1255       TypesToRoutingInfo(model_types),
1256       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1257       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1258   scheduler()->ScheduleConfiguration(params);
1259
1260   scheduler()->OnConnectionStatusChange();
1261   scheduler()->OnConnectionStatusChange();
1262
1263   PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
1264 }
1265
1266 TEST_F(SyncSchedulerTest, PollFromCanaryAfterAuthError) {
1267   SyncShareTimes times;
1268   TimeDelta poll(TimeDelta::FromMilliseconds(15));
1269   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1270
1271   ::testing::InSequence seq;
1272   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1273       .WillRepeatedly(
1274           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1275                 RecordSyncShareMultiple(&times, kMinNumSamples)));
1276
1277   connection()->SetServerStatus(HttpResponse::SYNC_AUTH_ERROR);
1278   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1279
1280   // Run to wait for polling.
1281   RunLoop();
1282
1283   // Normally OnCredentialsUpdated calls TryCanaryJob that doesn't run Poll,
1284   // but after poll finished with auth error from poll timer it should retry
1285   // poll once more
1286   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1287       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1288                       RecordSyncShare(&times)));
1289   scheduler()->OnCredentialsUpdated();
1290   connection()->SetServerStatus(HttpResponse::SERVER_CONNECTION_OK);
1291   RunLoop();
1292   StopSyncScheduler();
1293 }
1294
1295 TEST_F(SyncSchedulerTest, SuccessfulRetry) {
1296   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1297
1298   SyncShareTimes times;
1299   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(10);
1300   scheduler()->OnReceivedGuRetryDelay(delay);
1301   EXPECT_EQ(delay, GetRetryTimerDelay());
1302
1303   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1304       .WillOnce(
1305           DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1306                 RecordSyncShare(&times)));
1307
1308   // Run to wait for retrying.
1309   RunLoop();
1310
1311   StopSyncScheduler();
1312 }
1313
1314 TEST_F(SyncSchedulerTest, FailedRetry) {
1315   UseMockDelayProvider();
1316   EXPECT_CALL(*delay(), GetDelay(_))
1317       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(10)));
1318
1319   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1320
1321   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(10);
1322   scheduler()->OnReceivedGuRetryDelay(delay);
1323
1324   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1325       .WillOnce(
1326           DoAll(Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
1327                 QuitLoopNowAction()));
1328
1329   // Run to wait for retrying.
1330   RunLoop();
1331
1332   EXPECT_TRUE(scheduler()->IsBackingOff());
1333   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1334       .WillOnce(
1335           DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1336                 QuitLoopNowAction()));
1337
1338   // Run to wait for second retrying.
1339   RunLoop();
1340
1341   StopSyncScheduler();
1342 }
1343
1344 ACTION_P2(VerifyRetryTimerDelay, scheduler_test, expected_delay) {
1345   EXPECT_EQ(expected_delay, scheduler_test->GetRetryTimerDelay());
1346 }
1347
1348 TEST_F(SyncSchedulerTest, ReceiveNewRetryDelay) {
1349   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1350
1351   SyncShareTimes times;
1352   base::TimeDelta delay1 = base::TimeDelta::FromMilliseconds(100);
1353   base::TimeDelta delay2 = base::TimeDelta::FromMilliseconds(200);
1354
1355   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1356   scheduler()->OnReceivedGuRetryDelay(delay1);
1357   EXPECT_EQ(delay1, GetRetryTimerDelay());
1358
1359   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1360       .WillOnce(DoAll(
1361           WithoutArgs(VerifyRetryTimerDelay(this, delay1)),
1362           WithArg<2>(sessions::test_util::SimulateGuRetryDelayCommand(delay2)),
1363           RecordSyncShare(&times)));
1364
1365   // Run nudge GU.
1366   RunLoop();
1367   EXPECT_EQ(delay2, GetRetryTimerDelay());
1368
1369   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1370       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1371                       RecordSyncShare(&times)));
1372
1373   // Run to wait for retrying.
1374   RunLoop();
1375
1376   StopSyncScheduler();
1377 }
1378
1379 }  // namespace syncer