sync/engine/sync_scheduler_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/bind.h"
   6 #include "base/callback.h"
   7 #include "base/compiler_specific.h"
   8 #include "base/memory/weak_ptr.h"
   9 #include "base/message_loop/message_loop.h"
  10 #include "base/test/test_timeouts.h"
  11 #include "sync/engine/backoff_delay_provider.h"
  12 #include "sync/engine/sync_scheduler_impl.h"
  13 #include "sync/engine/syncer.h"
  14 #include "sync/internal_api/public/base/cancelation_signal.h"
  15 #include "sync/internal_api/public/base/model_type_test_util.h"
  16 #include "sync/notifier/invalidation_util.h"
  17 #include "sync/notifier/object_id_invalidation_map.h"
  18 #include "sync/sessions/test_util.h"
  19 #include "sync/test/callback_counter.h"
  20 #include "sync/test/engine/fake_model_worker.h"
  21 #include "sync/test/engine/mock_connection_manager.h"
  22 #include "sync/test/engine/test_directory_setter_upper.h"
  23 #include "sync/util/extensions_activity.h"
  24 #include "testing/gmock/include/gmock/gmock.h"
  25 #include "testing/gtest/include/gtest/gtest.h"
  26
  27 using base::TimeDelta;
  28 using base::TimeTicks;
  29 using testing::_;
  30 using testing::AtLeast;
  31 using testing::DoAll;
  32 using testing::Invoke;
  33 using testing::Mock;
  34 using testing::Return;
  35 using testing::WithArg;
  36 using testing::WithArgs;
  37 using testing::WithoutArgs;
  38
  39 namespace syncer {
  40 using sessions::SyncSession;
  41 using sessions::SyncSessionContext;
  42 using sync_pb::GetUpdatesCallerInfo;
  43
  44 class MockSyncer : public Syncer {
  45  public:
  46   MockSyncer();
  47   MOCK_METHOD3(NormalSyncShare, bool(ModelTypeSet,
  48                                      const sessions::NudgeTracker&,
  49                                      sessions::SyncSession*));
  50   MOCK_METHOD3(ConfigureSyncShare,
  51                bool(ModelTypeSet,
  52                     sync_pb::GetUpdatesCallerInfo::GetUpdatesSource,
  53                     SyncSession*));
  54   MOCK_METHOD2(PollSyncShare, bool(ModelTypeSet, sessions::SyncSession*));
  55 };
  56
  57 MockSyncer::MockSyncer()
  58   : Syncer(NULL) {}
  59
  60 typedef std::vector<TimeTicks> SyncShareTimes;
  61
  62 void QuitLoopNow() {
  63   // We use QuitNow() instead of Quit() as the latter may get stalled
  64   // indefinitely in the presence of repeated timers with low delays
  65   // and a slow test (e.g., ThrottlingDoesThrottle [which has a poll
  66   // delay of 5ms] run under TSAN on the trybots).
  67   base::MessageLoop::current()->QuitNow();
  68 }
  69
  70 void RunLoop() {
  71   base::MessageLoop::current()->Run();
  72 }
  73
  74 void PumpLoop() {
  75   // Do it this way instead of RunAllPending to pump loop exactly once
  76   // (necessary in the presence of timers; see comment in
  77   // QuitLoopNow).
  78   base::MessageLoop::current()->PostTask(FROM_HERE, base::Bind(&QuitLoopNow));
  79   RunLoop();
  80 }
  81
  82 void PumpLoopFor(base::TimeDelta time) {
  83   // Allow the loop to run for the specified amount of time.
  84   base::MessageLoop::current()->PostDelayedTask(
  85       FROM_HERE, base::Bind(&QuitLoopNow), time);
  86   RunLoop();
  87 }
  88
  89 ModelSafeRoutingInfo TypesToRoutingInfo(ModelTypeSet types) {
  90   ModelSafeRoutingInfo routes;
  91   for (ModelTypeSet::Iterator iter = types.First(); iter.Good(); iter.Inc()) {
  92     routes[iter.Get()] = GROUP_PASSIVE;
  93   }
  94   return routes;
  95 }
  96
  97 // Convenient to use in tests wishing to analyze SyncShare calls over time.
  98 static const size_t kMinNumSamples = 5;
  99 class SyncSchedulerTest : public testing::Test {
 100  public:
 101   SyncSchedulerTest() : syncer_(NULL), delay_(NULL), weak_ptr_factory_(this) {}
 102
 103   class MockDelayProvider : public BackoffDelayProvider {
 104    public:
 105     MockDelayProvider() : BackoffDelayProvider(
 106         TimeDelta::FromSeconds(kInitialBackoffRetrySeconds),
 107         TimeDelta::FromSeconds(kInitialBackoffImmediateRetrySeconds)) {
 108     }
 109
 110     MOCK_METHOD1(GetDelay, TimeDelta(const TimeDelta&));
 111   };
 112
 113   virtual void SetUp() {
 114     dir_maker_.SetUp();
 115     syncer_ = new testing::StrictMock<MockSyncer>();
 116     delay_ = NULL;
 117     extensions_activity_ = new ExtensionsActivity();
 118
 119     routing_info_[BOOKMARKS] = GROUP_UI;
 120     routing_info_[AUTOFILL] = GROUP_DB;
 121     routing_info_[THEMES] = GROUP_UI;
 122     routing_info_[NIGORI] = GROUP_PASSIVE;
 123
 124     workers_.clear();
 125     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_UI)));
 126     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_DB)));
 127     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_PASSIVE)));
 128
 129     connection_.reset(new MockConnectionManager(directory(),
 130                                                 &cancelation_signal_));
 131     connection_->SetServerReachable();
 132
 133     model_type_registry_.reset(new ModelTypeRegistry(workers_, directory()));
 134
 135     context_.reset(new SyncSessionContext(
 136             connection_.get(), directory(),
 137             extensions_activity_.get(),
 138             std::vector<SyncEngineEventListener*>(), NULL,
 139             model_type_registry_.get(),
 140             true,  // enable keystore encryption
 141             false,  // force enable pre-commit GU avoidance
 142             "fake_invalidator_client_id"));
 143     context_->SetRoutingInfo(routing_info_);
 144     context_->set_notifications_enabled(true);
 145     context_->set_account_name("Test");
 146     scheduler_.reset(
 147         new SyncSchedulerImpl("TestSyncScheduler",
 148             BackoffDelayProvider::FromDefaults(),
 149             context(),
 150             syncer_));
 151   }
 152
 153   SyncSchedulerImpl* scheduler() { return scheduler_.get(); }
 154   const ModelSafeRoutingInfo& routing_info() { return routing_info_; }
 155   MockSyncer* syncer() { return syncer_; }
 156   MockDelayProvider* delay() { return delay_; }
 157   MockConnectionManager* connection() { return connection_.get(); }
 158   TimeDelta zero() { return TimeDelta::FromSeconds(0); }
 159   TimeDelta timeout() {
 160     return TestTimeouts::action_timeout();
 161   }
 162
 163   virtual void TearDown() {
 164     PumpLoop();
 165     scheduler_.reset();
 166     PumpLoop();
 167     dir_maker_.TearDown();
 168   }
 169
 170   void AnalyzePollRun(const SyncShareTimes& times, size_t min_num_samples,
 171       const TimeTicks& optimal_start, const TimeDelta& poll_interval) {
 172     EXPECT_GE(times.size(), min_num_samples);
 173     for (size_t i = 0; i < times.size(); i++) {
 174       SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
 175       TimeTicks optimal_next_sync = optimal_start + poll_interval * i;
 176       EXPECT_GE(times[i], optimal_next_sync);
 177     }
 178   }
 179
 180   void DoQuitLoopNow() {
 181     QuitLoopNow();
 182   }
 183
 184   void StartSyncScheduler(SyncScheduler::Mode mode) {
 185     scheduler()->Start(mode);
 186   }
 187
 188   // This stops the scheduler synchronously.
 189   void StopSyncScheduler() {
 190     base::MessageLoop::current()->PostTask(
 191         FROM_HERE,
 192         base::Bind(&SyncSchedulerTest::DoQuitLoopNow,
 193                    weak_ptr_factory_.GetWeakPtr()));
 194     RunLoop();
 195   }
 196
 197   bool RunAndGetBackoff() {
 198     ModelTypeSet nudge_types(BOOKMARKS);
 199     StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 200
 201     scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
 202     RunLoop();
 203
 204     return scheduler()->IsBackingOff();
 205   }
 206
 207   void UseMockDelayProvider() {
 208     delay_ = new MockDelayProvider();
 209     scheduler_->delay_provider_.reset(delay_);
 210   }
 211
 212   SyncSessionContext* context() { return context_.get(); }
 213
 214   ModelTypeSet GetThrottledTypes() {
 215     return scheduler_->nudge_tracker_.GetThrottledTypes();
 216   }
 217
 218   base::TimeDelta GetRetryTimerDelay() {
 219     EXPECT_TRUE(scheduler_->retry_timer_.IsRunning());
 220     return scheduler_->retry_timer_.GetCurrentDelay();
 221   }
 222
 223  private:
 224   syncable::Directory* directory() {
 225     return dir_maker_.directory();
 226   }
 227
 228   base::MessageLoop loop_;
 229   TestDirectorySetterUpper dir_maker_;
 230   CancelationSignal cancelation_signal_;
 231   scoped_ptr<MockConnectionManager> connection_;
 232   scoped_ptr<ModelTypeRegistry> model_type_registry_;
 233   scoped_ptr<SyncSessionContext> context_;
 234   scoped_ptr<SyncSchedulerImpl> scheduler_;
 235   MockSyncer* syncer_;
 236   MockDelayProvider* delay_;
 237   std::vector<scoped_refptr<ModelSafeWorker> > workers_;
 238   scoped_refptr<ExtensionsActivity> extensions_activity_;
 239   ModelSafeRoutingInfo routing_info_;
 240   base::WeakPtrFactory<SyncSchedulerTest> weak_ptr_factory_;
 241 };
 242
 243 void RecordSyncShareImpl(SyncShareTimes* times) {
 244   times->push_back(TimeTicks::Now());
 245 }
 246
 247 ACTION_P(RecordSyncShare, times) {
 248   RecordSyncShareImpl(times);
 249   if (base::MessageLoop::current()->is_running())
 250     QuitLoopNow();
 251   return true;
 252 }
 253
 254 ACTION_P2(RecordSyncShareMultiple, times, quit_after) {
 255   RecordSyncShareImpl(times);
 256   EXPECT_LE(times->size(), quit_after);
 257   if (times->size() >= quit_after &&
 258       base::MessageLoop::current()->is_running()) {
 259     QuitLoopNow();
 260   }
 261   return true;
 262 }
 263
 264 ACTION_P(StopScheduler, scheduler) {
 265   scheduler->Stop();
 266 }
 267
 268 ACTION(AddFailureAndQuitLoopNow) {
 269   ADD_FAILURE();
 270   QuitLoopNow();
 271   return true;
 272 }
 273
 274 ACTION(QuitLoopNowAction) {
 275   QuitLoopNow();
 276   return true;
 277 }
 278
 279 // Test nudge scheduling.
 280 TEST_F(SyncSchedulerTest, Nudge) {
 281   SyncShareTimes times;
 282   ModelTypeSet model_types(BOOKMARKS);
 283
 284   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 285       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 286                       RecordSyncShare(&times)))
 287       .RetiresOnSaturation();
 288
 289   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 290
 291   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 292   RunLoop();
 293
 294   Mock::VerifyAndClearExpectations(syncer());
 295
 296   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 297   SyncShareTimes times2;
 298   model_types.Remove(BOOKMARKS);
 299   model_types.Put(AUTOFILL);
 300   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 301       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 302                       RecordSyncShare(&times2)));
 303   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 304   RunLoop();
 305 }
 306
 307 // Make sure a regular config command is scheduled fine in the absence of any
 308 // errors.
 309 TEST_F(SyncSchedulerTest, Config) {
 310   SyncShareTimes times;
 311   const ModelTypeSet model_types(BOOKMARKS);
 312
 313   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 314       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 315                       RecordSyncShare(&times)));
 316
 317   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 318
 319   CallbackCounter ready_counter;
 320   CallbackCounter retry_counter;
 321   ConfigurationParams params(
 322       GetUpdatesCallerInfo::RECONFIGURATION,
 323       model_types,
 324       TypesToRoutingInfo(model_types),
 325       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 326       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 327   scheduler()->ScheduleConfiguration(params);
 328   PumpLoop();
 329   ASSERT_EQ(1, ready_counter.times_called());
 330   ASSERT_EQ(0, retry_counter.times_called());
 331 }
 332
 333 // Simulate a failure and make sure the config request is retried.
 334 TEST_F(SyncSchedulerTest, ConfigWithBackingOff) {
 335   UseMockDelayProvider();
 336   EXPECT_CALL(*delay(), GetDelay(_))
 337       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
 338   SyncShareTimes times;
 339   const ModelTypeSet model_types(BOOKMARKS);
 340
 341   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 342
 343   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 344       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 345                       RecordSyncShare(&times)))
 346       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 347                       RecordSyncShare(&times)));
 348
 349   CallbackCounter ready_counter;
 350   CallbackCounter retry_counter;
 351   ConfigurationParams params(
 352       GetUpdatesCallerInfo::RECONFIGURATION,
 353       model_types,
 354       TypesToRoutingInfo(model_types),
 355       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 356       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 357   scheduler()->ScheduleConfiguration(params);
 358   RunLoop();
 359   ASSERT_EQ(0, ready_counter.times_called());
 360   ASSERT_EQ(1, retry_counter.times_called());
 361
 362   // RunLoop() will trigger TryCanaryJob which will retry configuration.
 363   // Since retry_task was already called it shouldn't be called again.
 364   RunLoop();
 365   ASSERT_EQ(0, ready_counter.times_called());
 366   ASSERT_EQ(1, retry_counter.times_called());
 367
 368   Mock::VerifyAndClearExpectations(syncer());
 369
 370   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 371       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 372                       RecordSyncShare(&times)));
 373   RunLoop();
 374
 375   ASSERT_EQ(1, ready_counter.times_called());
 376 }
 377
 378 // Simuilate SyncSchedulerImpl::Stop being called in the middle of Configure.
 379 // This can happen if server returns NOT_MY_BIRTHDAY.
 380 TEST_F(SyncSchedulerTest, ConfigWithStop) {
 381   UseMockDelayProvider();
 382   EXPECT_CALL(*delay(), GetDelay(_))
 383       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
 384   SyncShareTimes times;
 385   const ModelTypeSet model_types(BOOKMARKS);
 386
 387   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 388
 389   // Make ConfigureSyncShare call scheduler->Stop(). It is not supposed to call
 390   // retry_task or dereference configuration params.
 391   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 392       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 393                       StopScheduler(scheduler()),
 394                       RecordSyncShare(&times)));
 395
 396   CallbackCounter ready_counter;
 397   CallbackCounter retry_counter;
 398   ConfigurationParams params(
 399       GetUpdatesCallerInfo::RECONFIGURATION,
 400       model_types,
 401       TypesToRoutingInfo(model_types),
 402       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 403       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 404   scheduler()->ScheduleConfiguration(params);
 405   PumpLoop();
 406   ASSERT_EQ(0, ready_counter.times_called());
 407   ASSERT_EQ(0, retry_counter.times_called());
 408 }
 409
 410 // Issue a nudge when the config has failed. Make sure both the config and
 411 // nudge are executed.
 412 TEST_F(SyncSchedulerTest, NudgeWithConfigWithBackingOff) {
 413   const ModelTypeSet model_types(BOOKMARKS);
 414   UseMockDelayProvider();
 415   EXPECT_CALL(*delay(), GetDelay(_))
 416       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(50)));
 417   SyncShareTimes times;
 418
 419   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 420
 421   // Request a configure and make sure it fails.
 422   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 423       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 424                       RecordSyncShare(&times)));
 425   CallbackCounter ready_counter;
 426   CallbackCounter retry_counter;
 427   ConfigurationParams params(
 428       GetUpdatesCallerInfo::RECONFIGURATION,
 429       model_types,
 430       TypesToRoutingInfo(model_types),
 431       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 432       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 433   scheduler()->ScheduleConfiguration(params);
 434   RunLoop();
 435   ASSERT_EQ(0, ready_counter.times_called());
 436   ASSERT_EQ(1, retry_counter.times_called());
 437   Mock::VerifyAndClearExpectations(syncer());
 438
 439   // Ask for a nudge while dealing with repeated configure failure.
 440   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 441       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 442                       RecordSyncShare(&times)));
 443   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 444   RunLoop();
 445   // Note that we're not RunLoop()ing for the NUDGE we just scheduled, but
 446   // for the first retry attempt from the config job (after
 447   // waiting ~+/- 50ms).
 448   Mock::VerifyAndClearExpectations(syncer());
 449   ASSERT_EQ(0, ready_counter.times_called());
 450
 451   // Let the next configure retry succeed.
 452   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 453       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 454                       RecordSyncShare(&times)));
 455   RunLoop();
 456
 457   // Now change the mode so nudge can execute.
 458   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 459       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 460                       RecordSyncShare(&times)));
 461   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 462   PumpLoop();
 463 }
 464
 465 // Test that nudges are coalesced.
 466 TEST_F(SyncSchedulerTest, NudgeCoalescing) {
 467   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 468
 469   SyncShareTimes times;
 470   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 471       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 472                       RecordSyncShare(&times)));
 473   const ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3(THEMES);
 474   TimeDelta delay = zero();
 475   TimeTicks optimal_time = TimeTicks::Now() + delay;
 476   scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
 477   scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
 478   RunLoop();
 479
 480   ASSERT_EQ(1U, times.size());
 481   EXPECT_GE(times[0], optimal_time);
 482
 483   Mock::VerifyAndClearExpectations(syncer());
 484
 485   SyncShareTimes times2;
 486   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 487       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 488                       RecordSyncShare(&times2)));
 489   scheduler()->ScheduleLocalNudge(zero(), types3, FROM_HERE);
 490   RunLoop();
 491 }
 492
 493 // Test that nudges are coalesced.
 494 TEST_F(SyncSchedulerTest, NudgeCoalescingWithDifferentTimings) {
 495   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 496
 497   SyncShareTimes times;
 498   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 499       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 500                       RecordSyncShare(&times)));
 501   ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3;
 502
 503   // Create a huge time delay.
 504   TimeDelta delay = TimeDelta::FromDays(1);
 505
 506   scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
 507   scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
 508
 509   TimeTicks min_time = TimeTicks::Now();
 510   TimeTicks max_time = TimeTicks::Now() + delay;
 511
 512   RunLoop();
 513   Mock::VerifyAndClearExpectations(syncer());
 514
 515   // Make sure the sync happened at the right time.
 516   ASSERT_EQ(1U, times.size());
 517   EXPECT_GE(times[0], min_time);
 518   EXPECT_LE(times[0], max_time);
 519 }
 520
 521 // Test nudge scheduling.
 522 TEST_F(SyncSchedulerTest, NudgeWithStates) {
 523   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 524
 525   SyncShareTimes times1;
 526   ObjectIdInvalidationMap invalidations1 =
 527       BuildInvalidationMap(BOOKMARKS, 10, "test");
 528   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 529       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 530                       RecordSyncShare(&times1)))
 531       .RetiresOnSaturation();
 532   scheduler()->ScheduleInvalidationNudge(zero(), invalidations1, FROM_HERE);
 533   RunLoop();
 534
 535   Mock::VerifyAndClearExpectations(syncer());
 536
 537   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 538   SyncShareTimes times2;
 539   ObjectIdInvalidationMap invalidations2 =
 540       BuildInvalidationMap(AUTOFILL, 10, "test2");
 541   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 542       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 543                       RecordSyncShare(&times2)));
 544   scheduler()->ScheduleInvalidationNudge(zero(), invalidations2, FROM_HERE);
 545   RunLoop();
 546 }
 547
 548 // Test that polling works as expected.
 549 TEST_F(SyncSchedulerTest, Polling) {
 550   SyncShareTimes times;
 551   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 552   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 553       .WillRepeatedly(
 554           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 555                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 556
 557   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
 558
 559   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 560   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 561
 562   // Run again to wait for polling.
 563   RunLoop();
 564
 565   StopSyncScheduler();
 566   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 567 }
 568
 569 // Test that the short poll interval is used.
 570 TEST_F(SyncSchedulerTest, PollNotificationsDisabled) {
 571   SyncShareTimes times;
 572   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 573   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 574       .WillRepeatedly(
 575           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 576                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 577
 578   scheduler()->OnReceivedShortPollIntervalUpdate(poll_interval);
 579   scheduler()->SetNotificationsEnabled(false);
 580
 581   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 582   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 583
 584   // Run again to wait for polling.
 585   RunLoop();
 586
 587   StopSyncScheduler();
 588   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 589 }
 590
 591 // Test that polling intervals are updated when needed.
 592 TEST_F(SyncSchedulerTest, PollIntervalUpdate) {
 593   SyncShareTimes times;
 594   TimeDelta poll1(TimeDelta::FromMilliseconds(120));
 595   TimeDelta poll2(TimeDelta::FromMilliseconds(30));
 596   scheduler()->OnReceivedLongPollIntervalUpdate(poll1);
 597   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 598       .WillOnce(DoAll(
 599           WithArgs<0,1>(
 600               sessions::test_util::SimulatePollIntervalUpdate(poll2)),
 601           Return(true)))
 602       .WillRepeatedly(
 603           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 604                 WithArg<1>(
 605                     RecordSyncShareMultiple(&times, kMinNumSamples))));
 606
 607   TimeTicks optimal_start = TimeTicks::Now() + poll1 + poll2;
 608   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 609
 610   // Run again to wait for polling.
 611   RunLoop();
 612
 613   StopSyncScheduler();
 614   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll2);
 615 }
 616
 617 // Test that the sessions commit delay is updated when needed.
 618 TEST_F(SyncSchedulerTest, SessionsCommitDelay) {
 619   SyncShareTimes times;
 620   TimeDelta delay1(TimeDelta::FromMilliseconds(120));
 621   TimeDelta delay2(TimeDelta::FromMilliseconds(30));
 622   scheduler()->OnReceivedSessionsCommitDelay(delay1);
 623
 624   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 625       .WillOnce(
 626           DoAll(
 627               WithArgs<0,1,2>(
 628                   sessions::test_util::SimulateSessionsCommitDelayUpdate(
 629                       delay2)),
 630               Invoke(sessions::test_util::SimulateNormalSuccess),
 631               QuitLoopNowAction()));
 632
 633   EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
 634   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 635
 636   EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
 637   const ModelTypeSet model_types(BOOKMARKS);
 638   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 639   RunLoop();
 640
 641   EXPECT_EQ(delay2, scheduler()->GetSessionsCommitDelay());
 642   StopSyncScheduler();
 643 }
 644
 645 // Test that no syncing occurs when throttled.
 646 TEST_F(SyncSchedulerTest, ThrottlingDoesThrottle) {
 647   const ModelTypeSet types(BOOKMARKS);
 648   TimeDelta poll(TimeDelta::FromMilliseconds(5));
 649   TimeDelta throttle(TimeDelta::FromMinutes(10));
 650   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 651
 652   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 653       .WillOnce(DoAll(
 654           WithArg<2>(sessions::test_util::SimulateThrottled(throttle)),
 655           Return(true)))
 656       .WillRepeatedly(AddFailureAndQuitLoopNow());
 657
 658   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 659
 660   scheduler()->ScheduleLocalNudge(
 661       TimeDelta::FromMicroseconds(1), types, FROM_HERE);
 662   PumpLoop();
 663
 664   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 665
 666   CallbackCounter ready_counter;
 667   CallbackCounter retry_counter;
 668   ConfigurationParams params(
 669       GetUpdatesCallerInfo::RECONFIGURATION,
 670       types,
 671       TypesToRoutingInfo(types),
 672       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 673       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 674   scheduler()->ScheduleConfiguration(params);
 675   PumpLoop();
 676   ASSERT_EQ(0, ready_counter.times_called());
 677   ASSERT_EQ(1, retry_counter.times_called());
 678
 679 }
 680
 681 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromPoll) {
 682   SyncShareTimes times;
 683   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 684   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 685   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 686
 687   ::testing::InSequence seq;
 688   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 689       .WillOnce(DoAll(
 690           WithArg<1>(sessions::test_util::SimulateThrottled(throttle1)),
 691           Return(true)))
 692       .RetiresOnSaturation();
 693   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 694       .WillRepeatedly(
 695           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 696                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 697
 698   TimeTicks optimal_start = TimeTicks::Now() + poll + throttle1;
 699   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 700
 701   // Run again to wait for polling.
 702   RunLoop();
 703
 704   StopSyncScheduler();
 705   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll);
 706 }
 707
 708 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromNudge) {
 709   SyncShareTimes times;
 710   TimeDelta poll(TimeDelta::FromDays(1));
 711   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 712   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 713
 714   ::testing::InSequence seq;
 715   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 716       .WillOnce(DoAll(
 717           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 718           Return(true)))
 719       .RetiresOnSaturation();
 720   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 721       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 722                       QuitLoopNowAction()));
 723
 724   const ModelTypeSet types(BOOKMARKS);
 725   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 726   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 727
 728   PumpLoop(); // To get PerformDelayedNudge called.
 729   PumpLoop(); // To get TrySyncSessionJob called
 730   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 731   RunLoop();
 732   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 733
 734   StopSyncScheduler();
 735 }
 736
 737 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromConfigure) {
 738   SyncShareTimes times;
 739   TimeDelta poll(TimeDelta::FromDays(1));
 740   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 741   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 742
 743   ::testing::InSequence seq;
 744   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 745       .WillOnce(DoAll(
 746           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 747           Return(true)))
 748       .RetiresOnSaturation();
 749   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 750       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 751                       QuitLoopNowAction()));
 752
 753   const ModelTypeSet types(BOOKMARKS);
 754   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 755
 756   CallbackCounter ready_counter;
 757   CallbackCounter retry_counter;
 758   ConfigurationParams params(
 759       GetUpdatesCallerInfo::RECONFIGURATION,
 760       types,
 761       TypesToRoutingInfo(types),
 762       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 763       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 764   scheduler()->ScheduleConfiguration(params);
 765   PumpLoop();
 766   EXPECT_EQ(0, ready_counter.times_called());
 767   EXPECT_EQ(1, retry_counter.times_called());
 768   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 769
 770   RunLoop();
 771   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 772
 773   StopSyncScheduler();
 774 }
 775
 776 TEST_F(SyncSchedulerTest, TypeThrottlingBlocksNudge) {
 777   UseMockDelayProvider();
 778   EXPECT_CALL(*delay(), GetDelay(_))
 779       .WillRepeatedly(Return(zero()));
 780
 781   TimeDelta poll(TimeDelta::FromDays(1));
 782   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 783   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 784
 785   const ModelTypeSet types(BOOKMARKS);
 786
 787   ::testing::InSequence seq;
 788   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 789       .WillOnce(DoAll(
 790           WithArg<2>(
 791               sessions::test_util::SimulateTypesThrottled(types, throttle1)),
 792           Return(true)))
 793       .RetiresOnSaturation();
 794
 795   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 796   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 797   PumpLoop(); // To get PerformDelayedNudge called.
 798   PumpLoop(); // To get TrySyncSessionJob called
 799   EXPECT_TRUE(GetThrottledTypes().HasAll(types));
 800
 801   // This won't cause a sync cycle because the types are throttled.
 802   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 803   PumpLoop();
 804
 805   StopSyncScheduler();
 806 }
 807
 808 TEST_F(SyncSchedulerTest, TypeThrottlingDoesBlockOtherSources) {
 809   UseMockDelayProvider();
 810   EXPECT_CALL(*delay(), GetDelay(_))
 811       .WillRepeatedly(Return(zero()));
 812
 813   SyncShareTimes times;
 814   TimeDelta poll(TimeDelta::FromDays(1));
 815   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 816   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 817
 818   const ModelTypeSet throttled_types(BOOKMARKS);
 819   const ModelTypeSet unthrottled_types(PREFERENCES);
 820
 821   ::testing::InSequence seq;
 822   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 823       .WillOnce(DoAll(
 824           WithArg<2>(
 825               sessions::test_util::SimulateTypesThrottled(
 826                   throttled_types, throttle1)),
 827           Return(true)))
 828       .RetiresOnSaturation();
 829
 830   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 831   scheduler()->ScheduleLocalNudge(zero(), throttled_types, FROM_HERE);
 832   PumpLoop(); // To get PerformDelayedNudge called.
 833   PumpLoop(); // To get TrySyncSessionJob called
 834   EXPECT_TRUE(GetThrottledTypes().HasAll(throttled_types));
 835
 836   // Ignore invalidations for throttled types.
 837   ObjectIdInvalidationMap invalidations =
 838       BuildInvalidationMap(BOOKMARKS, 10, "test");
 839   scheduler()->ScheduleInvalidationNudge(zero(), invalidations, FROM_HERE);
 840   PumpLoop();
 841
 842   // Ignore refresh requests for throttled types.
 843   scheduler()->ScheduleLocalRefreshRequest(zero(), throttled_types, FROM_HERE);
 844   PumpLoop();
 845
 846   Mock::VerifyAndClearExpectations(syncer());
 847
 848   // Local nudges for non-throttled types will trigger a sync.
 849   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 850       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 851                             RecordSyncShare(&times)));
 852   scheduler()->ScheduleLocalNudge(zero(), unthrottled_types, FROM_HERE);
 853   RunLoop();
 854   Mock::VerifyAndClearExpectations(syncer());
 855
 856   StopSyncScheduler();
 857 }
 858
 859 // Test nudges / polls don't run in config mode and config tasks do.
 860 TEST_F(SyncSchedulerTest, ConfigurationMode) {
 861   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 862   SyncShareTimes times;
 863   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 864
 865   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 866
 867   const ModelTypeSet nudge_types(AUTOFILL);
 868   scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
 869   scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
 870
 871   const ModelTypeSet config_types(BOOKMARKS);
 872
 873   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 874       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 875                       RecordSyncShare(&times)))
 876       .RetiresOnSaturation();
 877   CallbackCounter ready_counter;
 878   CallbackCounter retry_counter;
 879   ConfigurationParams params(
 880       GetUpdatesCallerInfo::RECONFIGURATION,
 881       config_types,
 882       TypesToRoutingInfo(config_types),
 883       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 884       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 885   scheduler()->ScheduleConfiguration(params);
 886   RunLoop();
 887   ASSERT_EQ(1, ready_counter.times_called());
 888   ASSERT_EQ(0, retry_counter.times_called());
 889
 890   Mock::VerifyAndClearExpectations(syncer());
 891
 892   // Switch to NORMAL_MODE to ensure NUDGES were properly saved and run.
 893   scheduler()->OnReceivedLongPollIntervalUpdate(TimeDelta::FromDays(1));
 894   SyncShareTimes times2;
 895   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 896       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 897                       RecordSyncShare(&times2)));
 898
 899   // TODO(tim): Figure out how to remove this dangerous need to reset
 900   // routing info between mode switches.
 901   context()->SetRoutingInfo(routing_info());
 902   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 903
 904   RunLoop();
 905   Mock::VerifyAndClearExpectations(syncer());
 906 }
 907
 908 class BackoffTriggersSyncSchedulerTest : public SyncSchedulerTest {
 909   virtual void SetUp() {
 910     SyncSchedulerTest::SetUp();
 911     UseMockDelayProvider();
 912     EXPECT_CALL(*delay(), GetDelay(_))
 913         .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
 914   }
 915
 916   virtual void TearDown() {
 917     StopSyncScheduler();
 918     SyncSchedulerTest::TearDown();
 919   }
 920 };
 921
 922 // Have the sycner fail during commit.  Expect that the scheduler enters
 923 // backoff.
 924 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnce) {
 925   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 926       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
 927                       QuitLoopNowAction()));
 928   EXPECT_TRUE(RunAndGetBackoff());
 929 }
 930
 931 // Have the syncer fail during download updates and succeed on the first
 932 // retry.  Expect that this clears the backoff state.
 933 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadOnceThenSucceed) {
 934   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 935       .WillOnce(DoAll(
 936           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 937           Return(true)))
 938       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 939                       QuitLoopNowAction()));
 940   EXPECT_FALSE(RunAndGetBackoff());
 941 }
 942
 943 // Have the syncer fail during commit and succeed on the first retry.  Expect
 944 // that this clears the backoff state.
 945 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnceThenSucceed) {
 946   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 947       .WillOnce(DoAll(
 948           Invoke(sessions::test_util::SimulateCommitFailed),
 949           Return(true)))
 950       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 951                       QuitLoopNowAction()));
 952   EXPECT_FALSE(RunAndGetBackoff());
 953 }
 954
 955 // Have the syncer fail to download updates and fail again on the retry.
 956 // Expect this will leave the scheduler in backoff.
 957 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadTwice) {
 958   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 959       .WillOnce(DoAll(
 960           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 961           Return(true)))
 962       .WillRepeatedly(DoAll(
 963               Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 964               QuitLoopNowAction()));
 965   EXPECT_TRUE(RunAndGetBackoff());
 966 }
 967
 968 // Have the syncer fail to get the encryption key yet succeed in downloading
 969 // updates. Expect this will leave the scheduler in backoff.
 970 TEST_F(BackoffTriggersSyncSchedulerTest, FailGetEncryptionKey) {
 971   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 972       .WillOnce(DoAll(
 973           Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
 974           Return(true)))
 975       .WillRepeatedly(DoAll(
 976               Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
 977               QuitLoopNowAction()));
 978   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 979
 980   ModelTypeSet types(BOOKMARKS);
 981   CallbackCounter ready_counter;
 982   CallbackCounter retry_counter;
 983   ConfigurationParams params(
 984       GetUpdatesCallerInfo::RECONFIGURATION,
 985       types,
 986       TypesToRoutingInfo(types),
 987       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 988       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 989   scheduler()->ScheduleConfiguration(params);
 990   RunLoop();
 991
 992   EXPECT_TRUE(scheduler()->IsBackingOff());
 993 }
 994
 995 // Test that no polls or extraneous nudges occur when in backoff.
 996 TEST_F(SyncSchedulerTest, BackoffDropsJobs) {
 997   SyncShareTimes times;
 998   TimeDelta poll(TimeDelta::FromMilliseconds(5));
 999   const ModelTypeSet types(BOOKMARKS);
1000   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1001   UseMockDelayProvider();
1002
1003   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1004       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1005                       RecordSyncShareMultiple(&times, 1U)));
1006   EXPECT_CALL(*delay(), GetDelay(_)).
1007       WillRepeatedly(Return(TimeDelta::FromDays(1)));
1008
1009   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1010
1011   // This nudge should fail and put us into backoff.  Thanks to our mock
1012   // GetDelay() setup above, this will be a long backoff.
1013   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
1014   RunLoop();
1015
1016   // From this point forward, no SyncShare functions should be invoked.
1017   Mock::VerifyAndClearExpectations(syncer());
1018
1019   // Wait a while (10x poll interval) so a few poll jobs will be attempted.
1020   PumpLoopFor(poll * 10);
1021
1022   // Try (and fail) to schedule a nudge.
1023   scheduler()->ScheduleLocalNudge(
1024       base::TimeDelta::FromMilliseconds(1),
1025       types,
1026       FROM_HERE);
1027
1028   Mock::VerifyAndClearExpectations(syncer());
1029   Mock::VerifyAndClearExpectations(delay());
1030
1031   EXPECT_CALL(*delay(), GetDelay(_)).Times(0);
1032
1033   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1034
1035   CallbackCounter ready_counter;
1036   CallbackCounter retry_counter;
1037   ConfigurationParams params(
1038       GetUpdatesCallerInfo::RECONFIGURATION,
1039       types,
1040       TypesToRoutingInfo(types),
1041       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1042       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1043   scheduler()->ScheduleConfiguration(params);
1044   PumpLoop();
1045   ASSERT_EQ(0, ready_counter.times_called());
1046   ASSERT_EQ(1, retry_counter.times_called());
1047
1048 }
1049
1050 // Test that backoff is shaping traffic properly with consecutive errors.
1051 TEST_F(SyncSchedulerTest, BackoffElevation) {
1052   SyncShareTimes times;
1053   UseMockDelayProvider();
1054
1055   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_)).Times(kMinNumSamples)
1056       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1057           RecordSyncShareMultiple(&times, kMinNumSamples)));
1058
1059   const TimeDelta first = TimeDelta::FromSeconds(kInitialBackoffRetrySeconds);
1060   const TimeDelta second = TimeDelta::FromMilliseconds(2);
1061   const TimeDelta third = TimeDelta::FromMilliseconds(3);
1062   const TimeDelta fourth = TimeDelta::FromMilliseconds(4);
1063   const TimeDelta fifth = TimeDelta::FromMilliseconds(5);
1064   const TimeDelta sixth = TimeDelta::FromDays(1);
1065
1066   EXPECT_CALL(*delay(), GetDelay(first)).WillOnce(Return(second))
1067           .RetiresOnSaturation();
1068   EXPECT_CALL(*delay(), GetDelay(second)).WillOnce(Return(third))
1069           .RetiresOnSaturation();
1070   EXPECT_CALL(*delay(), GetDelay(third)).WillOnce(Return(fourth))
1071           .RetiresOnSaturation();
1072   EXPECT_CALL(*delay(), GetDelay(fourth)).WillOnce(Return(fifth))
1073           .RetiresOnSaturation();
1074   EXPECT_CALL(*delay(), GetDelay(fifth)).WillOnce(Return(sixth));
1075
1076   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1077
1078   // Run again with a nudge.
1079   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1080   RunLoop();
1081
1082   ASSERT_EQ(kMinNumSamples, times.size());
1083   EXPECT_GE(times[1] - times[0], second);
1084   EXPECT_GE(times[2] - times[1], third);
1085   EXPECT_GE(times[3] - times[2], fourth);
1086   EXPECT_GE(times[4] - times[3], fifth);
1087 }
1088
1089 // Test that things go back to normal once a retry makes forward progress.
1090 TEST_F(SyncSchedulerTest, BackoffRelief) {
1091   SyncShareTimes times;
1092   const TimeDelta poll(TimeDelta::FromMilliseconds(10));
1093   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1094   UseMockDelayProvider();
1095
1096   const TimeDelta backoff = TimeDelta::FromMilliseconds(5);
1097   EXPECT_CALL(*delay(), GetDelay(_)).WillOnce(Return(backoff));
1098
1099   // Optimal start for the post-backoff poll party.
1100   TimeTicks optimal_start = TimeTicks::Now();
1101   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1102
1103   // Kick off the test with a failed nudge.
1104   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1105       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1106                       RecordSyncShare(&times)));
1107   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1108   RunLoop();
1109   Mock::VerifyAndClearExpectations(syncer());
1110   TimeTicks optimal_job_time = optimal_start;
1111   ASSERT_EQ(1U, times.size());
1112   EXPECT_GE(times[0], optimal_job_time);
1113
1114   // The retry succeeds.
1115   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1116       .WillOnce(DoAll(
1117               Invoke(sessions::test_util::SimulateNormalSuccess),
1118               RecordSyncShare(&times)));
1119   RunLoop();
1120   Mock::VerifyAndClearExpectations(syncer());
1121   optimal_job_time = optimal_job_time + backoff;
1122   ASSERT_EQ(2U, times.size());
1123   EXPECT_GE(times[1], optimal_job_time);
1124
1125   // Now let the Poll timer do its thing.
1126   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1127       .WillRepeatedly(DoAll(
1128               Invoke(sessions::test_util::SimulatePollSuccess),
1129               RecordSyncShareMultiple(&times, kMinNumSamples)));
1130   RunLoop();
1131   Mock::VerifyAndClearExpectations(syncer());
1132   ASSERT_EQ(kMinNumSamples, times.size());
1133   for (size_t i = 2; i < times.size(); i++) {
1134     optimal_job_time = optimal_job_time + poll;
1135     SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
1136     EXPECT_GE(times[i], optimal_job_time);
1137   }
1138
1139   StopSyncScheduler();
1140 }
1141
1142 // Test that poll failures are ignored.  They should have no effect on
1143 // subsequent poll attempts, nor should they trigger a backoff/retry.
1144 TEST_F(SyncSchedulerTest, TransientPollFailure) {
1145   SyncShareTimes times;
1146   const TimeDelta poll_interval(TimeDelta::FromMilliseconds(1));
1147   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
1148   UseMockDelayProvider(); // Will cause test failure if backoff is initiated.
1149
1150   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1151       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollFailed),
1152                       RecordSyncShare(&times)))
1153       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1154                       RecordSyncShare(&times)));
1155
1156   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1157
1158   // Run the unsucessful poll. The failed poll should not trigger backoff.
1159   RunLoop();
1160   EXPECT_FALSE(scheduler()->IsBackingOff());
1161
1162   // Run the successful poll.
1163   RunLoop();
1164   EXPECT_FALSE(scheduler()->IsBackingOff());
1165 }
1166
1167 // Test that starting the syncer thread without a valid connection doesn't
1168 // break things when a connection is detected.
1169 TEST_F(SyncSchedulerTest, StartWhenNotConnected) {
1170   connection()->SetServerNotReachable();
1171   connection()->UpdateConnectionStatus();
1172   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1173     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1174                     Return(true)))
1175     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1176                     Return(true)));
1177   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1178
1179   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1180   // Should save the nudge for until after the server is reachable.
1181   base::MessageLoop::current()->RunUntilIdle();
1182
1183   scheduler()->OnConnectionStatusChange();
1184   connection()->SetServerReachable();
1185   connection()->UpdateConnectionStatus();
1186   base::MessageLoop::current()->RunUntilIdle();
1187 }
1188
1189 TEST_F(SyncSchedulerTest, ServerConnectionChangeDuringBackoff) {
1190   UseMockDelayProvider();
1191   EXPECT_CALL(*delay(), GetDelay(_))
1192       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1193
1194   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1195   connection()->SetServerNotReachable();
1196   connection()->UpdateConnectionStatus();
1197
1198   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1199     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1200                     Return(true)))
1201     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1202                     Return(true)));
1203
1204   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1205   PumpLoop(); // To get PerformDelayedNudge called.
1206   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1207   ASSERT_TRUE(scheduler()->IsBackingOff());
1208
1209   // Before we run the scheduled canary, trigger a server connection change.
1210   scheduler()->OnConnectionStatusChange();
1211   connection()->SetServerReachable();
1212   connection()->UpdateConnectionStatus();
1213   base::MessageLoop::current()->RunUntilIdle();
1214 }
1215
1216 // This was supposed to test the scenario where we receive a nudge while a
1217 // connection change canary is scheduled, but has not run yet.  Since we've made
1218 // the connection change canary synchronous, this is no longer possible.
1219 TEST_F(SyncSchedulerTest, ConnectionChangeCanaryPreemptedByNudge) {
1220   UseMockDelayProvider();
1221   EXPECT_CALL(*delay(), GetDelay(_))
1222       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1223
1224   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1225   connection()->SetServerNotReachable();
1226   connection()->UpdateConnectionStatus();
1227
1228   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1229     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1230                     Return(true)))
1231     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1232                     Return(true)))
1233     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1234                     QuitLoopNowAction()));
1235
1236   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1237
1238   PumpLoop(); // To get PerformDelayedNudge called.
1239   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1240   ASSERT_TRUE(scheduler()->IsBackingOff());
1241
1242   // Before we run the scheduled canary, trigger a server connection change.
1243   scheduler()->OnConnectionStatusChange();
1244   PumpLoop();
1245   connection()->SetServerReachable();
1246   connection()->UpdateConnectionStatus();
1247   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1248   base::MessageLoop::current()->RunUntilIdle();
1249 }
1250
1251 // Tests that we don't crash trying to run two canaries at once if we receive
1252 // extra connection status change notifications.  See crbug.com/190085.
1253 TEST_F(SyncSchedulerTest, DoubleCanaryInConfigure) {
1254   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
1255       .WillRepeatedly(DoAll(
1256               Invoke(sessions::test_util::SimulateConfigureConnectionFailure),
1257               Return(true)));
1258   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1259   connection()->SetServerNotReachable();
1260   connection()->UpdateConnectionStatus();
1261
1262   ModelTypeSet model_types(BOOKMARKS);
1263   CallbackCounter ready_counter;
1264   CallbackCounter retry_counter;
1265   ConfigurationParams params(
1266       GetUpdatesCallerInfo::RECONFIGURATION,
1267       model_types,
1268       TypesToRoutingInfo(model_types),
1269       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1270       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1271   scheduler()->ScheduleConfiguration(params);
1272
1273   scheduler()->OnConnectionStatusChange();
1274   scheduler()->OnConnectionStatusChange();
1275
1276   PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
1277 }
1278
1279 TEST_F(SyncSchedulerTest, PollFromCanaryAfterAuthError) {
1280   SyncShareTimes times;
1281   TimeDelta poll(TimeDelta::FromMilliseconds(15));
1282   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1283
1284   ::testing::InSequence seq;
1285   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1286       .WillRepeatedly(
1287           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1288                 RecordSyncShareMultiple(&times, kMinNumSamples)));
1289
1290   connection()->SetServerStatus(HttpResponse::SYNC_AUTH_ERROR);
1291   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1292
1293   // Run to wait for polling.
1294   RunLoop();
1295
1296   // Normally OnCredentialsUpdated calls TryCanaryJob that doesn't run Poll,
1297   // but after poll finished with auth error from poll timer it should retry
1298   // poll once more
1299   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1300       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1301                       RecordSyncShare(&times)));
1302   scheduler()->OnCredentialsUpdated();
1303   connection()->SetServerStatus(HttpResponse::SERVER_CONNECTION_OK);
1304   RunLoop();
1305   StopSyncScheduler();
1306 }
1307
1308 TEST_F(SyncSchedulerTest, SuccessfulRetry) {
1309   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1310
1311   SyncShareTimes times;
1312   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(1);
1313   scheduler()->OnReceivedGuRetryDelay(delay);
1314   EXPECT_EQ(delay, GetRetryTimerDelay());
1315
1316   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1317       .WillOnce(
1318           DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1319                 RecordSyncShare(&times)));
1320
1321   // Run to wait for retrying.
1322   RunLoop();
1323
1324   StopSyncScheduler();
1325 }
1326
1327 TEST_F(SyncSchedulerTest, FailedRetry) {
1328   UseMockDelayProvider();
1329   EXPECT_CALL(*delay(), GetDelay(_))
1330       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
1331
1332   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1333
1334   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(1);
1335   scheduler()->OnReceivedGuRetryDelay(delay);
1336
1337   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1338       .WillOnce(
1339           DoAll(Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
1340                 QuitLoopNowAction()));
1341
1342   // Run to wait for retrying.
1343   RunLoop();
1344
1345   EXPECT_TRUE(scheduler()->IsBackingOff());
1346   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1347       .WillOnce(
1348           DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1349                 QuitLoopNowAction()));
1350
1351   // Run to wait for second retrying.
1352   RunLoop();
1353
1354   StopSyncScheduler();
1355 }
1356
1357 ACTION_P2(VerifyRetryTimerDelay, scheduler_test, expected_delay) {
1358   EXPECT_EQ(expected_delay, scheduler_test->GetRetryTimerDelay());
1359 }
1360
1361 TEST_F(SyncSchedulerTest, ReceiveNewRetryDelay) {
1362   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1363
1364   SyncShareTimes times;
1365   base::TimeDelta delay1 = base::TimeDelta::FromMilliseconds(100);
1366   base::TimeDelta delay2 = base::TimeDelta::FromMilliseconds(200);
1367
1368   scheduler()->ScheduleLocalRefreshRequest(zero(), ModelTypeSet(BOOKMARKS),
1369                                            FROM_HERE);
1370   scheduler()->OnReceivedGuRetryDelay(delay1);
1371   EXPECT_EQ(delay1, GetRetryTimerDelay());
1372
1373   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1374       .WillOnce(DoAll(
1375           WithoutArgs(VerifyRetryTimerDelay(this, delay1)),
1376           WithArg<2>(sessions::test_util::SimulateGuRetryDelayCommand(delay2)),
1377           RecordSyncShare(&times)));
1378
1379   // Run nudge GU.
1380   RunLoop();
1381   EXPECT_EQ(delay2, GetRetryTimerDelay());
1382
1383   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1384       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1385                       RecordSyncShare(&times)));
1386
1387   // Run to wait for retrying.
1388   RunLoop();
1389
1390   StopSyncScheduler();
1391 }
1392
1393 }  // namespace syncer