sync/engine/sync_scheduler_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/bind.h"
   6 #include "base/callback.h"
   7 #include "base/compiler_specific.h"
   8 #include "base/memory/weak_ptr.h"
   9 #include "base/message_loop/message_loop.h"
  10 #include "base/test/test_timeouts.h"
  11 #include "sync/engine/backoff_delay_provider.h"
  12 #include "sync/engine/sync_scheduler_impl.h"
  13 #include "sync/engine/syncer.h"
  14 #include "sync/internal_api/public/base/cancelation_signal.h"
  15 #include "sync/internal_api/public/base/model_type_test_util.h"
  16 #include "sync/notifier/invalidation_util.h"
  17 #include "sync/notifier/object_id_invalidation_map.h"
  18 #include "sync/sessions/test_util.h"
  19 #include "sync/test/callback_counter.h"
  20 #include "sync/test/engine/fake_model_worker.h"
  21 #include "sync/test/engine/mock_connection_manager.h"
  22 #include "sync/test/engine/test_directory_setter_upper.h"
  23 #include "sync/util/extensions_activity.h"
  24 #include "testing/gmock/include/gmock/gmock.h"
  25 #include "testing/gtest/include/gtest/gtest.h"
  26
  27 using base::TimeDelta;
  28 using base::TimeTicks;
  29 using testing::_;
  30 using testing::AtLeast;
  31 using testing::DoAll;
  32 using testing::Invoke;
  33 using testing::Mock;
  34 using testing::Return;
  35 using testing::WithArg;
  36 using testing::WithArgs;
  37 using testing::WithoutArgs;
  38
  39 namespace syncer {
  40 using sessions::SyncSession;
  41 using sessions::SyncSessionContext;
  42 using sync_pb::GetUpdatesCallerInfo;
  43
  44 class MockSyncer : public Syncer {
  45  public:
  46   MockSyncer();
  47   MOCK_METHOD3(NormalSyncShare, bool(ModelTypeSet,
  48                                      const sessions::NudgeTracker&,
  49                                      sessions::SyncSession*));
  50   MOCK_METHOD3(ConfigureSyncShare,
  51                bool(ModelTypeSet,
  52                     sync_pb::GetUpdatesCallerInfo::GetUpdatesSource,
  53                     SyncSession*));
  54   MOCK_METHOD2(PollSyncShare, bool(ModelTypeSet, sessions::SyncSession*));
  55   MOCK_METHOD2(RetrySyncShare, bool(ModelTypeSet, sessions::SyncSession*));
  56 };
  57
  58 MockSyncer::MockSyncer()
  59   : Syncer(NULL) {}
  60
  61 typedef std::vector<TimeTicks> SyncShareTimes;
  62
  63 void QuitLoopNow() {
  64   // We use QuitNow() instead of Quit() as the latter may get stalled
  65   // indefinitely in the presence of repeated timers with low delays
  66   // and a slow test (e.g., ThrottlingDoesThrottle [which has a poll
  67   // delay of 5ms] run under TSAN on the trybots).
  68   base::MessageLoop::current()->QuitNow();
  69 }
  70
  71 void RunLoop() {
  72   base::MessageLoop::current()->Run();
  73 }
  74
  75 void PumpLoop() {
  76   // Do it this way instead of RunAllPending to pump loop exactly once
  77   // (necessary in the presence of timers; see comment in
  78   // QuitLoopNow).
  79   base::MessageLoop::current()->PostTask(FROM_HERE, base::Bind(&QuitLoopNow));
  80   RunLoop();
  81 }
  82
  83 void PumpLoopFor(base::TimeDelta time) {
  84   // Allow the loop to run for the specified amount of time.
  85   base::MessageLoop::current()->PostDelayedTask(
  86       FROM_HERE, base::Bind(&QuitLoopNow), time);
  87   RunLoop();
  88 }
  89
  90 ModelSafeRoutingInfo TypesToRoutingInfo(ModelTypeSet types) {
  91   ModelSafeRoutingInfo routes;
  92   for (ModelTypeSet::Iterator iter = types.First(); iter.Good(); iter.Inc()) {
  93     routes[iter.Get()] = GROUP_PASSIVE;
  94   }
  95   return routes;
  96 }
  97
  98 // Convenient to use in tests wishing to analyze SyncShare calls over time.
  99 static const size_t kMinNumSamples = 5;
 100 class SyncSchedulerTest : public testing::Test {
 101  public:
 102   SyncSchedulerTest() : syncer_(NULL), delay_(NULL), weak_ptr_factory_(this) {}
 103
 104   class MockDelayProvider : public BackoffDelayProvider {
 105    public:
 106     MockDelayProvider() : BackoffDelayProvider(
 107         TimeDelta::FromSeconds(kInitialBackoffRetrySeconds),
 108         TimeDelta::FromSeconds(kInitialBackoffImmediateRetrySeconds)) {
 109     }
 110
 111     MOCK_METHOD1(GetDelay, TimeDelta(const TimeDelta&));
 112   };
 113
 114   virtual void SetUp() {
 115     dir_maker_.SetUp();
 116     syncer_ = new testing::StrictMock<MockSyncer>();
 117     delay_ = NULL;
 118     extensions_activity_ = new ExtensionsActivity();
 119
 120     routing_info_[BOOKMARKS] = GROUP_UI;
 121     routing_info_[AUTOFILL] = GROUP_DB;
 122     routing_info_[THEMES] = GROUP_UI;
 123     routing_info_[NIGORI] = GROUP_PASSIVE;
 124
 125     workers_.clear();
 126     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_UI)));
 127     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_DB)));
 128     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_PASSIVE)));
 129
 130     connection_.reset(new MockConnectionManager(directory(),
 131                                                 &cancelation_signal_));
 132     connection_->SetServerReachable();
 133
 134     model_type_registry_.reset(new ModelTypeRegistry(workers_, directory()));
 135
 136     context_.reset(new SyncSessionContext(
 137             connection_.get(), directory(),
 138             extensions_activity_.get(),
 139             std::vector<SyncEngineEventListener*>(), NULL, NULL,
 140             model_type_registry_.get(),
 141             true,  // enable keystore encryption
 142             false,  // force enable pre-commit GU avoidance
 143             "fake_invalidator_client_id"));
 144     context_->SetRoutingInfo(routing_info_);
 145     context_->set_notifications_enabled(true);
 146     context_->set_account_name("Test");
 147     scheduler_.reset(
 148         new SyncSchedulerImpl("TestSyncScheduler",
 149             BackoffDelayProvider::FromDefaults(),
 150             context(),
 151             syncer_));
 152   }
 153
 154   SyncSchedulerImpl* scheduler() { return scheduler_.get(); }
 155   const ModelSafeRoutingInfo& routing_info() { return routing_info_; }
 156   MockSyncer* syncer() { return syncer_; }
 157   MockDelayProvider* delay() { return delay_; }
 158   MockConnectionManager* connection() { return connection_.get(); }
 159   TimeDelta zero() { return TimeDelta::FromSeconds(0); }
 160   TimeDelta timeout() {
 161     return TestTimeouts::action_timeout();
 162   }
 163
 164   virtual void TearDown() {
 165     PumpLoop();
 166     scheduler_.reset();
 167     PumpLoop();
 168     dir_maker_.TearDown();
 169   }
 170
 171   void AnalyzePollRun(const SyncShareTimes& times, size_t min_num_samples,
 172       const TimeTicks& optimal_start, const TimeDelta& poll_interval) {
 173     EXPECT_GE(times.size(), min_num_samples);
 174     for (size_t i = 0; i < times.size(); i++) {
 175       SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
 176       TimeTicks optimal_next_sync = optimal_start + poll_interval * i;
 177       EXPECT_GE(times[i], optimal_next_sync);
 178     }
 179   }
 180
 181   void DoQuitLoopNow() {
 182     QuitLoopNow();
 183   }
 184
 185   void StartSyncScheduler(SyncScheduler::Mode mode) {
 186     scheduler()->Start(mode);
 187   }
 188
 189   // This stops the scheduler synchronously.
 190   void StopSyncScheduler() {
 191     base::MessageLoop::current()->PostTask(
 192         FROM_HERE,
 193         base::Bind(&SyncSchedulerTest::DoQuitLoopNow,
 194                    weak_ptr_factory_.GetWeakPtr()));
 195     RunLoop();
 196   }
 197
 198   bool RunAndGetBackoff() {
 199     ModelTypeSet nudge_types(BOOKMARKS);
 200     StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 201
 202     scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
 203     RunLoop();
 204
 205     return scheduler()->IsBackingOff();
 206   }
 207
 208   void UseMockDelayProvider() {
 209     delay_ = new MockDelayProvider();
 210     scheduler_->delay_provider_.reset(delay_);
 211   }
 212
 213   SyncSessionContext* context() { return context_.get(); }
 214
 215   ModelTypeSet GetThrottledTypes() {
 216     return scheduler_->nudge_tracker_.GetThrottledTypes();
 217   }
 218
 219   base::TimeDelta GetRetryTimerDelay() {
 220     EXPECT_TRUE(scheduler_->retry_timer_.IsRunning());
 221     return scheduler_->retry_timer_.GetCurrentDelay();
 222   }
 223
 224  private:
 225   syncable::Directory* directory() {
 226     return dir_maker_.directory();
 227   }
 228
 229   base::MessageLoop loop_;
 230   TestDirectorySetterUpper dir_maker_;
 231   CancelationSignal cancelation_signal_;
 232   scoped_ptr<MockConnectionManager> connection_;
 233   scoped_ptr<ModelTypeRegistry> model_type_registry_;
 234   scoped_ptr<SyncSessionContext> context_;
 235   scoped_ptr<SyncSchedulerImpl> scheduler_;
 236   MockSyncer* syncer_;
 237   MockDelayProvider* delay_;
 238   std::vector<scoped_refptr<ModelSafeWorker> > workers_;
 239   scoped_refptr<ExtensionsActivity> extensions_activity_;
 240   ModelSafeRoutingInfo routing_info_;
 241   base::WeakPtrFactory<SyncSchedulerTest> weak_ptr_factory_;
 242 };
 243
 244 void RecordSyncShareImpl(SyncShareTimes* times) {
 245   times->push_back(TimeTicks::Now());
 246 }
 247
 248 ACTION_P(RecordSyncShare, times) {
 249   RecordSyncShareImpl(times);
 250   if (base::MessageLoop::current()->is_running())
 251     QuitLoopNow();
 252   return true;
 253 }
 254
 255 ACTION_P2(RecordSyncShareMultiple, times, quit_after) {
 256   RecordSyncShareImpl(times);
 257   EXPECT_LE(times->size(), quit_after);
 258   if (times->size() >= quit_after &&
 259       base::MessageLoop::current()->is_running()) {
 260     QuitLoopNow();
 261   }
 262   return true;
 263 }
 264
 265 ACTION_P(StopScheduler, scheduler) {
 266   scheduler->Stop();
 267 }
 268
 269 ACTION(AddFailureAndQuitLoopNow) {
 270   ADD_FAILURE();
 271   QuitLoopNow();
 272   return true;
 273 }
 274
 275 ACTION(QuitLoopNowAction) {
 276   QuitLoopNow();
 277   return true;
 278 }
 279
 280 // Test nudge scheduling.
 281 TEST_F(SyncSchedulerTest, Nudge) {
 282   SyncShareTimes times;
 283   ModelTypeSet model_types(BOOKMARKS);
 284
 285   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 286       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 287                       RecordSyncShare(&times)))
 288       .RetiresOnSaturation();
 289
 290   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 291
 292   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 293   RunLoop();
 294
 295   Mock::VerifyAndClearExpectations(syncer());
 296
 297   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 298   SyncShareTimes times2;
 299   model_types.Remove(BOOKMARKS);
 300   model_types.Put(AUTOFILL);
 301   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 302       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 303                       RecordSyncShare(&times2)));
 304   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 305   RunLoop();
 306 }
 307
 308 // Make sure a regular config command is scheduled fine in the absence of any
 309 // errors.
 310 TEST_F(SyncSchedulerTest, Config) {
 311   SyncShareTimes times;
 312   const ModelTypeSet model_types(BOOKMARKS);
 313
 314   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 315       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 316                       RecordSyncShare(&times)));
 317
 318   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 319
 320   CallbackCounter ready_counter;
 321   CallbackCounter retry_counter;
 322   ConfigurationParams params(
 323       GetUpdatesCallerInfo::RECONFIGURATION,
 324       model_types,
 325       TypesToRoutingInfo(model_types),
 326       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 327       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 328   scheduler()->ScheduleConfiguration(params);
 329   PumpLoop();
 330   ASSERT_EQ(1, ready_counter.times_called());
 331   ASSERT_EQ(0, retry_counter.times_called());
 332 }
 333
 334 // Simulate a failure and make sure the config request is retried.
 335 TEST_F(SyncSchedulerTest, ConfigWithBackingOff) {
 336   UseMockDelayProvider();
 337   EXPECT_CALL(*delay(), GetDelay(_))
 338       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
 339   SyncShareTimes times;
 340   const ModelTypeSet model_types(BOOKMARKS);
 341
 342   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 343
 344   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 345       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 346                       RecordSyncShare(&times)))
 347       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 348                       RecordSyncShare(&times)));
 349
 350   CallbackCounter ready_counter;
 351   CallbackCounter retry_counter;
 352   ConfigurationParams params(
 353       GetUpdatesCallerInfo::RECONFIGURATION,
 354       model_types,
 355       TypesToRoutingInfo(model_types),
 356       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 357       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 358   scheduler()->ScheduleConfiguration(params);
 359   RunLoop();
 360   ASSERT_EQ(0, ready_counter.times_called());
 361   ASSERT_EQ(1, retry_counter.times_called());
 362
 363   // RunLoop() will trigger TryCanaryJob which will retry configuration.
 364   // Since retry_task was already called it shouldn't be called again.
 365   RunLoop();
 366   ASSERT_EQ(0, ready_counter.times_called());
 367   ASSERT_EQ(1, retry_counter.times_called());
 368
 369   Mock::VerifyAndClearExpectations(syncer());
 370
 371   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 372       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 373                       RecordSyncShare(&times)));
 374   RunLoop();
 375
 376   ASSERT_EQ(1, ready_counter.times_called());
 377 }
 378
 379 // Simuilate SyncSchedulerImpl::Stop being called in the middle of Configure.
 380 // This can happen if server returns NOT_MY_BIRTHDAY.
 381 TEST_F(SyncSchedulerTest, ConfigWithStop) {
 382   UseMockDelayProvider();
 383   EXPECT_CALL(*delay(), GetDelay(_))
 384       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
 385   SyncShareTimes times;
 386   const ModelTypeSet model_types(BOOKMARKS);
 387
 388   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 389
 390   // Make ConfigureSyncShare call scheduler->Stop(). It is not supposed to call
 391   // retry_task or dereference configuration params.
 392   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 393       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 394                       StopScheduler(scheduler()),
 395                       RecordSyncShare(&times)));
 396
 397   CallbackCounter ready_counter;
 398   CallbackCounter retry_counter;
 399   ConfigurationParams params(
 400       GetUpdatesCallerInfo::RECONFIGURATION,
 401       model_types,
 402       TypesToRoutingInfo(model_types),
 403       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 404       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 405   scheduler()->ScheduleConfiguration(params);
 406   PumpLoop();
 407   ASSERT_EQ(0, ready_counter.times_called());
 408   ASSERT_EQ(0, retry_counter.times_called());
 409 }
 410
 411 // Issue a nudge when the config has failed. Make sure both the config and
 412 // nudge are executed.
 413 TEST_F(SyncSchedulerTest, NudgeWithConfigWithBackingOff) {
 414   const ModelTypeSet model_types(BOOKMARKS);
 415   UseMockDelayProvider();
 416   EXPECT_CALL(*delay(), GetDelay(_))
 417       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(50)));
 418   SyncShareTimes times;
 419
 420   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 421
 422   // Request a configure and make sure it fails.
 423   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 424       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 425                       RecordSyncShare(&times)));
 426   CallbackCounter ready_counter;
 427   CallbackCounter retry_counter;
 428   ConfigurationParams params(
 429       GetUpdatesCallerInfo::RECONFIGURATION,
 430       model_types,
 431       TypesToRoutingInfo(model_types),
 432       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 433       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 434   scheduler()->ScheduleConfiguration(params);
 435   RunLoop();
 436   ASSERT_EQ(0, ready_counter.times_called());
 437   ASSERT_EQ(1, retry_counter.times_called());
 438   Mock::VerifyAndClearExpectations(syncer());
 439
 440   // Ask for a nudge while dealing with repeated configure failure.
 441   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 442       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 443                       RecordSyncShare(&times)));
 444   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 445   RunLoop();
 446   // Note that we're not RunLoop()ing for the NUDGE we just scheduled, but
 447   // for the first retry attempt from the config job (after
 448   // waiting ~+/- 50ms).
 449   Mock::VerifyAndClearExpectations(syncer());
 450   ASSERT_EQ(0, ready_counter.times_called());
 451
 452   // Let the next configure retry succeed.
 453   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 454       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 455                       RecordSyncShare(&times)));
 456   RunLoop();
 457
 458   // Now change the mode so nudge can execute.
 459   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 460       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 461                       RecordSyncShare(&times)));
 462   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 463   PumpLoop();
 464 }
 465
 466 // Test that nudges are coalesced.
 467 TEST_F(SyncSchedulerTest, NudgeCoalescing) {
 468   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 469
 470   SyncShareTimes times;
 471   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 472       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 473                       RecordSyncShare(&times)));
 474   const ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3(THEMES);
 475   TimeDelta delay = zero();
 476   TimeTicks optimal_time = TimeTicks::Now() + delay;
 477   scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
 478   scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
 479   RunLoop();
 480
 481   ASSERT_EQ(1U, times.size());
 482   EXPECT_GE(times[0], optimal_time);
 483
 484   Mock::VerifyAndClearExpectations(syncer());
 485
 486   SyncShareTimes times2;
 487   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 488       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 489                       RecordSyncShare(&times2)));
 490   scheduler()->ScheduleLocalNudge(zero(), types3, FROM_HERE);
 491   RunLoop();
 492 }
 493
 494 // Test that nudges are coalesced.
 495 TEST_F(SyncSchedulerTest, NudgeCoalescingWithDifferentTimings) {
 496   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 497
 498   SyncShareTimes times;
 499   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 500       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 501                       RecordSyncShare(&times)));
 502   ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3;
 503
 504   // Create a huge time delay.
 505   TimeDelta delay = TimeDelta::FromDays(1);
 506
 507   scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
 508   scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
 509
 510   TimeTicks min_time = TimeTicks::Now();
 511   TimeTicks max_time = TimeTicks::Now() + delay;
 512
 513   RunLoop();
 514   Mock::VerifyAndClearExpectations(syncer());
 515
 516   // Make sure the sync happened at the right time.
 517   ASSERT_EQ(1U, times.size());
 518   EXPECT_GE(times[0], min_time);
 519   EXPECT_LE(times[0], max_time);
 520 }
 521
 522 // Test nudge scheduling.
 523 TEST_F(SyncSchedulerTest, NudgeWithStates) {
 524   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 525
 526   SyncShareTimes times1;
 527   ObjectIdInvalidationMap invalidations1 =
 528       BuildInvalidationMap(BOOKMARKS, 10, "test");
 529   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 530       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 531                       RecordSyncShare(&times1)))
 532       .RetiresOnSaturation();
 533   scheduler()->ScheduleInvalidationNudge(zero(), invalidations1, FROM_HERE);
 534   RunLoop();
 535
 536   Mock::VerifyAndClearExpectations(syncer());
 537
 538   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 539   SyncShareTimes times2;
 540   ObjectIdInvalidationMap invalidations2 =
 541       BuildInvalidationMap(AUTOFILL, 10, "test2");
 542   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 543       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 544                       RecordSyncShare(&times2)));
 545   scheduler()->ScheduleInvalidationNudge(zero(), invalidations2, FROM_HERE);
 546   RunLoop();
 547 }
 548
 549 // Test that polling works as expected.
 550 TEST_F(SyncSchedulerTest, Polling) {
 551   SyncShareTimes times;
 552   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 553   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 554       .WillRepeatedly(
 555           DoAll(Invoke(sessions::test_util::SimulatePollRetrySuccess),
 556                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 557
 558   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
 559
 560   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 561   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 562
 563   // Run again to wait for polling.
 564   RunLoop();
 565
 566   StopSyncScheduler();
 567   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 568 }
 569
 570 // Test that the short poll interval is used.
 571 TEST_F(SyncSchedulerTest, PollNotificationsDisabled) {
 572   SyncShareTimes times;
 573   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 574   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 575       .WillRepeatedly(
 576           DoAll(Invoke(sessions::test_util::SimulatePollRetrySuccess),
 577                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 578
 579   scheduler()->OnReceivedShortPollIntervalUpdate(poll_interval);
 580   scheduler()->SetNotificationsEnabled(false);
 581
 582   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 583   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 584
 585   // Run again to wait for polling.
 586   RunLoop();
 587
 588   StopSyncScheduler();
 589   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 590 }
 591
 592 // Test that polling intervals are updated when needed.
 593 TEST_F(SyncSchedulerTest, PollIntervalUpdate) {
 594   SyncShareTimes times;
 595   TimeDelta poll1(TimeDelta::FromMilliseconds(120));
 596   TimeDelta poll2(TimeDelta::FromMilliseconds(30));
 597   scheduler()->OnReceivedLongPollIntervalUpdate(poll1);
 598   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 599       .WillOnce(DoAll(
 600           WithArgs<0,1>(
 601               sessions::test_util::SimulatePollIntervalUpdate(poll2)),
 602           Return(true)))
 603       .WillRepeatedly(
 604           DoAll(Invoke(sessions::test_util::SimulatePollRetrySuccess),
 605                 WithArg<1>(
 606                     RecordSyncShareMultiple(&times, kMinNumSamples))));
 607
 608   TimeTicks optimal_start = TimeTicks::Now() + poll1 + poll2;
 609   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 610
 611   // Run again to wait for polling.
 612   RunLoop();
 613
 614   StopSyncScheduler();
 615   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll2);
 616 }
 617
 618 // Test that the sessions commit delay is updated when needed.
 619 TEST_F(SyncSchedulerTest, SessionsCommitDelay) {
 620   SyncShareTimes times;
 621   TimeDelta delay1(TimeDelta::FromMilliseconds(120));
 622   TimeDelta delay2(TimeDelta::FromMilliseconds(30));
 623   scheduler()->OnReceivedSessionsCommitDelay(delay1);
 624
 625   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 626       .WillOnce(
 627           DoAll(
 628               WithArgs<0,1,2>(
 629                   sessions::test_util::SimulateSessionsCommitDelayUpdate(
 630                       delay2)),
 631               Invoke(sessions::test_util::SimulateNormalSuccess),
 632               QuitLoopNowAction()));
 633
 634   EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
 635   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 636
 637   EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
 638   const ModelTypeSet model_types(BOOKMARKS);
 639   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 640   RunLoop();
 641
 642   EXPECT_EQ(delay2, scheduler()->GetSessionsCommitDelay());
 643   StopSyncScheduler();
 644 }
 645
 646 // Test that no syncing occurs when throttled.
 647 TEST_F(SyncSchedulerTest, ThrottlingDoesThrottle) {
 648   const ModelTypeSet types(BOOKMARKS);
 649   TimeDelta poll(TimeDelta::FromMilliseconds(5));
 650   TimeDelta throttle(TimeDelta::FromMinutes(10));
 651   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 652
 653   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 654       .WillOnce(DoAll(
 655           WithArg<2>(sessions::test_util::SimulateThrottled(throttle)),
 656           Return(true)))
 657       .WillRepeatedly(AddFailureAndQuitLoopNow());
 658
 659   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 660
 661   scheduler()->ScheduleLocalNudge(
 662       TimeDelta::FromMicroseconds(1), types, FROM_HERE);
 663   PumpLoop();
 664
 665   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 666
 667   CallbackCounter ready_counter;
 668   CallbackCounter retry_counter;
 669   ConfigurationParams params(
 670       GetUpdatesCallerInfo::RECONFIGURATION,
 671       types,
 672       TypesToRoutingInfo(types),
 673       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 674       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 675   scheduler()->ScheduleConfiguration(params);
 676   PumpLoop();
 677   ASSERT_EQ(0, ready_counter.times_called());
 678   ASSERT_EQ(1, retry_counter.times_called());
 679
 680 }
 681
 682 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromPoll) {
 683   SyncShareTimes times;
 684   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 685   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 686   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 687
 688   ::testing::InSequence seq;
 689   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 690       .WillOnce(DoAll(
 691           WithArg<1>(sessions::test_util::SimulateThrottled(throttle1)),
 692           Return(true)))
 693       .RetiresOnSaturation();
 694   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 695       .WillRepeatedly(
 696           DoAll(Invoke(sessions::test_util::SimulatePollRetrySuccess),
 697                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 698
 699   TimeTicks optimal_start = TimeTicks::Now() + poll + throttle1;
 700   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 701
 702   // Run again to wait for polling.
 703   RunLoop();
 704
 705   StopSyncScheduler();
 706   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll);
 707 }
 708
 709 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromNudge) {
 710   SyncShareTimes times;
 711   TimeDelta poll(TimeDelta::FromDays(1));
 712   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 713   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 714
 715   ::testing::InSequence seq;
 716   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 717       .WillOnce(DoAll(
 718           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 719           Return(true)))
 720       .RetiresOnSaturation();
 721   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 722       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 723                       QuitLoopNowAction()));
 724
 725   const ModelTypeSet types(BOOKMARKS);
 726   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 727   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 728
 729   PumpLoop(); // To get PerformDelayedNudge called.
 730   PumpLoop(); // To get TrySyncSessionJob called
 731   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 732   RunLoop();
 733   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 734
 735   StopSyncScheduler();
 736 }
 737
 738 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromConfigure) {
 739   SyncShareTimes times;
 740   TimeDelta poll(TimeDelta::FromDays(1));
 741   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 742   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 743
 744   ::testing::InSequence seq;
 745   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 746       .WillOnce(DoAll(
 747           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 748           Return(true)))
 749       .RetiresOnSaturation();
 750   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 751       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 752                       QuitLoopNowAction()));
 753
 754   const ModelTypeSet types(BOOKMARKS);
 755   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 756
 757   CallbackCounter ready_counter;
 758   CallbackCounter retry_counter;
 759   ConfigurationParams params(
 760       GetUpdatesCallerInfo::RECONFIGURATION,
 761       types,
 762       TypesToRoutingInfo(types),
 763       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 764       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 765   scheduler()->ScheduleConfiguration(params);
 766   PumpLoop();
 767   EXPECT_EQ(0, ready_counter.times_called());
 768   EXPECT_EQ(1, retry_counter.times_called());
 769   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 770
 771   RunLoop();
 772   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 773
 774   StopSyncScheduler();
 775 }
 776
 777 TEST_F(SyncSchedulerTest, TypeThrottlingBlocksNudge) {
 778   UseMockDelayProvider();
 779   EXPECT_CALL(*delay(), GetDelay(_))
 780       .WillRepeatedly(Return(zero()));
 781
 782   TimeDelta poll(TimeDelta::FromDays(1));
 783   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 784   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 785
 786   const ModelTypeSet types(BOOKMARKS);
 787
 788   ::testing::InSequence seq;
 789   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 790       .WillOnce(DoAll(
 791           WithArg<2>(
 792               sessions::test_util::SimulateTypesThrottled(types, throttle1)),
 793           Return(true)))
 794       .RetiresOnSaturation();
 795
 796   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 797   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 798   PumpLoop(); // To get PerformDelayedNudge called.
 799   PumpLoop(); // To get TrySyncSessionJob called
 800   EXPECT_TRUE(GetThrottledTypes().HasAll(types));
 801
 802   // This won't cause a sync cycle because the types are throttled.
 803   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 804   PumpLoop();
 805
 806   StopSyncScheduler();
 807 }
 808
 809 TEST_F(SyncSchedulerTest, TypeThrottlingDoesBlockOtherSources) {
 810   UseMockDelayProvider();
 811   EXPECT_CALL(*delay(), GetDelay(_))
 812       .WillRepeatedly(Return(zero()));
 813
 814   SyncShareTimes times;
 815   TimeDelta poll(TimeDelta::FromDays(1));
 816   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 817   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 818
 819   const ModelTypeSet throttled_types(BOOKMARKS);
 820   const ModelTypeSet unthrottled_types(PREFERENCES);
 821
 822   ::testing::InSequence seq;
 823   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 824       .WillOnce(DoAll(
 825           WithArg<2>(
 826               sessions::test_util::SimulateTypesThrottled(
 827                   throttled_types, throttle1)),
 828           Return(true)))
 829       .RetiresOnSaturation();
 830
 831   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 832   scheduler()->ScheduleLocalNudge(zero(), throttled_types, FROM_HERE);
 833   PumpLoop(); // To get PerformDelayedNudge called.
 834   PumpLoop(); // To get TrySyncSessionJob called
 835   EXPECT_TRUE(GetThrottledTypes().HasAll(throttled_types));
 836
 837   // Ignore invalidations for throttled types.
 838   ObjectIdInvalidationMap invalidations =
 839       BuildInvalidationMap(BOOKMARKS, 10, "test");
 840   scheduler()->ScheduleInvalidationNudge(zero(), invalidations, FROM_HERE);
 841   PumpLoop();
 842
 843   // Ignore refresh requests for throttled types.
 844   scheduler()->ScheduleLocalRefreshRequest(zero(), throttled_types, FROM_HERE);
 845   PumpLoop();
 846
 847   Mock::VerifyAndClearExpectations(syncer());
 848
 849   // Local nudges for non-throttled types will trigger a sync.
 850   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 851       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 852                             RecordSyncShare(&times)));
 853   scheduler()->ScheduleLocalNudge(zero(), unthrottled_types, FROM_HERE);
 854   RunLoop();
 855   Mock::VerifyAndClearExpectations(syncer());
 856
 857   StopSyncScheduler();
 858 }
 859
 860 // Test nudges / polls don't run in config mode and config tasks do.
 861 TEST_F(SyncSchedulerTest, ConfigurationMode) {
 862   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 863   SyncShareTimes times;
 864   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 865
 866   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 867
 868   const ModelTypeSet nudge_types(AUTOFILL);
 869   scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
 870   scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
 871
 872   const ModelTypeSet config_types(BOOKMARKS);
 873
 874   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 875       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 876                       RecordSyncShare(&times)))
 877       .RetiresOnSaturation();
 878   CallbackCounter ready_counter;
 879   CallbackCounter retry_counter;
 880   ConfigurationParams params(
 881       GetUpdatesCallerInfo::RECONFIGURATION,
 882       config_types,
 883       TypesToRoutingInfo(config_types),
 884       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 885       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 886   scheduler()->ScheduleConfiguration(params);
 887   RunLoop();
 888   ASSERT_EQ(1, ready_counter.times_called());
 889   ASSERT_EQ(0, retry_counter.times_called());
 890
 891   Mock::VerifyAndClearExpectations(syncer());
 892
 893   // Switch to NORMAL_MODE to ensure NUDGES were properly saved and run.
 894   scheduler()->OnReceivedLongPollIntervalUpdate(TimeDelta::FromDays(1));
 895   SyncShareTimes times2;
 896   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 897       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 898                       RecordSyncShare(&times2)));
 899
 900   // TODO(tim): Figure out how to remove this dangerous need to reset
 901   // routing info between mode switches.
 902   context()->SetRoutingInfo(routing_info());
 903   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 904
 905   RunLoop();
 906   Mock::VerifyAndClearExpectations(syncer());
 907 }
 908
 909 class BackoffTriggersSyncSchedulerTest : public SyncSchedulerTest {
 910   virtual void SetUp() {
 911     SyncSchedulerTest::SetUp();
 912     UseMockDelayProvider();
 913     EXPECT_CALL(*delay(), GetDelay(_))
 914         .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
 915   }
 916
 917   virtual void TearDown() {
 918     StopSyncScheduler();
 919     SyncSchedulerTest::TearDown();
 920   }
 921 };
 922
 923 // Have the sycner fail during commit.  Expect that the scheduler enters
 924 // backoff.
 925 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnce) {
 926   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 927       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
 928                       QuitLoopNowAction()));
 929   EXPECT_TRUE(RunAndGetBackoff());
 930 }
 931
 932 // Have the syncer fail during download updates and succeed on the first
 933 // retry.  Expect that this clears the backoff state.
 934 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadOnceThenSucceed) {
 935   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 936       .WillOnce(DoAll(
 937           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 938           Return(true)))
 939       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 940                       QuitLoopNowAction()));
 941   EXPECT_FALSE(RunAndGetBackoff());
 942 }
 943
 944 // Have the syncer fail during commit and succeed on the first retry.  Expect
 945 // that this clears the backoff state.
 946 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnceThenSucceed) {
 947   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 948       .WillOnce(DoAll(
 949           Invoke(sessions::test_util::SimulateCommitFailed),
 950           Return(true)))
 951       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 952                       QuitLoopNowAction()));
 953   EXPECT_FALSE(RunAndGetBackoff());
 954 }
 955
 956 // Have the syncer fail to download updates and fail again on the retry.
 957 // Expect this will leave the scheduler in backoff.
 958 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadTwice) {
 959   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 960       .WillOnce(DoAll(
 961           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 962           Return(true)))
 963       .WillRepeatedly(DoAll(
 964               Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 965               QuitLoopNowAction()));
 966   EXPECT_TRUE(RunAndGetBackoff());
 967 }
 968
 969 // Have the syncer fail to get the encryption key yet succeed in downloading
 970 // updates. Expect this will leave the scheduler in backoff.
 971 TEST_F(BackoffTriggersSyncSchedulerTest, FailGetEncryptionKey) {
 972   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 973       .WillOnce(DoAll(
 974           Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
 975           Return(true)))
 976       .WillRepeatedly(DoAll(
 977               Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
 978               QuitLoopNowAction()));
 979   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 980
 981   ModelTypeSet types(BOOKMARKS);
 982   CallbackCounter ready_counter;
 983   CallbackCounter retry_counter;
 984   ConfigurationParams params(
 985       GetUpdatesCallerInfo::RECONFIGURATION,
 986       types,
 987       TypesToRoutingInfo(types),
 988       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 989       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 990   scheduler()->ScheduleConfiguration(params);
 991   RunLoop();
 992
 993   EXPECT_TRUE(scheduler()->IsBackingOff());
 994 }
 995
 996 // Test that no polls or extraneous nudges occur when in backoff.
 997 TEST_F(SyncSchedulerTest, BackoffDropsJobs) {
 998   SyncShareTimes times;
 999   TimeDelta poll(TimeDelta::FromMilliseconds(5));
1000   const ModelTypeSet types(BOOKMARKS);
1001   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1002   UseMockDelayProvider();
1003
1004   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1005       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1006                       RecordSyncShareMultiple(&times, 1U)));
1007   EXPECT_CALL(*delay(), GetDelay(_)).
1008       WillRepeatedly(Return(TimeDelta::FromDays(1)));
1009
1010   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1011
1012   // This nudge should fail and put us into backoff.  Thanks to our mock
1013   // GetDelay() setup above, this will be a long backoff.
1014   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
1015   RunLoop();
1016
1017   // From this point forward, no SyncShare functions should be invoked.
1018   Mock::VerifyAndClearExpectations(syncer());
1019
1020   // Wait a while (10x poll interval) so a few poll jobs will be attempted.
1021   PumpLoopFor(poll * 10);
1022
1023   // Try (and fail) to schedule a nudge.
1024   scheduler()->ScheduleLocalNudge(
1025       base::TimeDelta::FromMilliseconds(1),
1026       types,
1027       FROM_HERE);
1028
1029   Mock::VerifyAndClearExpectations(syncer());
1030   Mock::VerifyAndClearExpectations(delay());
1031
1032   EXPECT_CALL(*delay(), GetDelay(_)).Times(0);
1033
1034   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1035
1036   CallbackCounter ready_counter;
1037   CallbackCounter retry_counter;
1038   ConfigurationParams params(
1039       GetUpdatesCallerInfo::RECONFIGURATION,
1040       types,
1041       TypesToRoutingInfo(types),
1042       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1043       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1044   scheduler()->ScheduleConfiguration(params);
1045   PumpLoop();
1046   ASSERT_EQ(0, ready_counter.times_called());
1047   ASSERT_EQ(1, retry_counter.times_called());
1048
1049 }
1050
1051 // Test that backoff is shaping traffic properly with consecutive errors.
1052 TEST_F(SyncSchedulerTest, BackoffElevation) {
1053   SyncShareTimes times;
1054   UseMockDelayProvider();
1055
1056   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_)).Times(kMinNumSamples)
1057       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1058           RecordSyncShareMultiple(&times, kMinNumSamples)));
1059
1060   const TimeDelta first = TimeDelta::FromSeconds(kInitialBackoffRetrySeconds);
1061   const TimeDelta second = TimeDelta::FromMilliseconds(2);
1062   const TimeDelta third = TimeDelta::FromMilliseconds(3);
1063   const TimeDelta fourth = TimeDelta::FromMilliseconds(4);
1064   const TimeDelta fifth = TimeDelta::FromMilliseconds(5);
1065   const TimeDelta sixth = TimeDelta::FromDays(1);
1066
1067   EXPECT_CALL(*delay(), GetDelay(first)).WillOnce(Return(second))
1068           .RetiresOnSaturation();
1069   EXPECT_CALL(*delay(), GetDelay(second)).WillOnce(Return(third))
1070           .RetiresOnSaturation();
1071   EXPECT_CALL(*delay(), GetDelay(third)).WillOnce(Return(fourth))
1072           .RetiresOnSaturation();
1073   EXPECT_CALL(*delay(), GetDelay(fourth)).WillOnce(Return(fifth))
1074           .RetiresOnSaturation();
1075   EXPECT_CALL(*delay(), GetDelay(fifth)).WillOnce(Return(sixth));
1076
1077   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1078
1079   // Run again with a nudge.
1080   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1081   RunLoop();
1082
1083   ASSERT_EQ(kMinNumSamples, times.size());
1084   EXPECT_GE(times[1] - times[0], second);
1085   EXPECT_GE(times[2] - times[1], third);
1086   EXPECT_GE(times[3] - times[2], fourth);
1087   EXPECT_GE(times[4] - times[3], fifth);
1088 }
1089
1090 // Test that things go back to normal once a retry makes forward progress.
1091 TEST_F(SyncSchedulerTest, BackoffRelief) {
1092   SyncShareTimes times;
1093   const TimeDelta poll(TimeDelta::FromMilliseconds(10));
1094   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1095   UseMockDelayProvider();
1096
1097   const TimeDelta backoff = TimeDelta::FromMilliseconds(5);
1098   EXPECT_CALL(*delay(), GetDelay(_)).WillOnce(Return(backoff));
1099
1100   // Optimal start for the post-backoff poll party.
1101   TimeTicks optimal_start = TimeTicks::Now();
1102   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1103
1104   // Kick off the test with a failed nudge.
1105   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1106       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1107                       RecordSyncShare(&times)));
1108   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1109   RunLoop();
1110   Mock::VerifyAndClearExpectations(syncer());
1111   TimeTicks optimal_job_time = optimal_start;
1112   ASSERT_EQ(1U, times.size());
1113   EXPECT_GE(times[0], optimal_job_time);
1114
1115   // The retry succeeds.
1116   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1117       .WillOnce(DoAll(
1118               Invoke(sessions::test_util::SimulateNormalSuccess),
1119               RecordSyncShare(&times)));
1120   RunLoop();
1121   Mock::VerifyAndClearExpectations(syncer());
1122   optimal_job_time = optimal_job_time + backoff;
1123   ASSERT_EQ(2U, times.size());
1124   EXPECT_GE(times[1], optimal_job_time);
1125
1126   // Now let the Poll timer do its thing.
1127   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1128       .WillRepeatedly(DoAll(
1129               Invoke(sessions::test_util::SimulatePollRetrySuccess),
1130               RecordSyncShareMultiple(&times, kMinNumSamples)));
1131   RunLoop();
1132   Mock::VerifyAndClearExpectations(syncer());
1133   ASSERT_EQ(kMinNumSamples, times.size());
1134   for (size_t i = 2; i < times.size(); i++) {
1135     optimal_job_time = optimal_job_time + poll;
1136     SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
1137     EXPECT_GE(times[i], optimal_job_time);
1138   }
1139
1140   StopSyncScheduler();
1141 }
1142
1143 // Test that poll failures are ignored.  They should have no effect on
1144 // subsequent poll attempts, nor should they trigger a backoff/retry.
1145 TEST_F(SyncSchedulerTest, TransientPollFailure) {
1146   SyncShareTimes times;
1147   const TimeDelta poll_interval(TimeDelta::FromMilliseconds(1));
1148   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
1149   UseMockDelayProvider(); // Will cause test failure if backoff is initiated.
1150
1151   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1152       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollRetryFailed),
1153                       RecordSyncShare(&times)))
1154       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollRetrySuccess),
1155                       RecordSyncShare(&times)));
1156
1157   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1158
1159   // Run the unsucessful poll. The failed poll should not trigger backoff.
1160   RunLoop();
1161   EXPECT_FALSE(scheduler()->IsBackingOff());
1162
1163   // Run the successful poll.
1164   RunLoop();
1165   EXPECT_FALSE(scheduler()->IsBackingOff());
1166 }
1167
1168 // Test that starting the syncer thread without a valid connection doesn't
1169 // break things when a connection is detected.
1170 TEST_F(SyncSchedulerTest, StartWhenNotConnected) {
1171   connection()->SetServerNotReachable();
1172   connection()->UpdateConnectionStatus();
1173   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1174     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1175                     Return(true)))
1176     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1177                     Return(true)));
1178   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1179
1180   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1181   // Should save the nudge for until after the server is reachable.
1182   base::MessageLoop::current()->RunUntilIdle();
1183
1184   scheduler()->OnConnectionStatusChange();
1185   connection()->SetServerReachable();
1186   connection()->UpdateConnectionStatus();
1187   base::MessageLoop::current()->RunUntilIdle();
1188 }
1189
1190 TEST_F(SyncSchedulerTest, ServerConnectionChangeDuringBackoff) {
1191   UseMockDelayProvider();
1192   EXPECT_CALL(*delay(), GetDelay(_))
1193       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1194
1195   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1196   connection()->SetServerNotReachable();
1197   connection()->UpdateConnectionStatus();
1198
1199   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1200     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1201                     Return(true)))
1202     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1203                     Return(true)));
1204
1205   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1206   PumpLoop(); // To get PerformDelayedNudge called.
1207   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1208   ASSERT_TRUE(scheduler()->IsBackingOff());
1209
1210   // Before we run the scheduled canary, trigger a server connection change.
1211   scheduler()->OnConnectionStatusChange();
1212   connection()->SetServerReachable();
1213   connection()->UpdateConnectionStatus();
1214   base::MessageLoop::current()->RunUntilIdle();
1215 }
1216
1217 // This was supposed to test the scenario where we receive a nudge while a
1218 // connection change canary is scheduled, but has not run yet.  Since we've made
1219 // the connection change canary synchronous, this is no longer possible.
1220 TEST_F(SyncSchedulerTest, ConnectionChangeCanaryPreemptedByNudge) {
1221   UseMockDelayProvider();
1222   EXPECT_CALL(*delay(), GetDelay(_))
1223       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1224
1225   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1226   connection()->SetServerNotReachable();
1227   connection()->UpdateConnectionStatus();
1228
1229   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1230     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1231                     Return(true)))
1232     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1233                     Return(true)))
1234     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1235                     QuitLoopNowAction()));
1236
1237   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1238
1239   PumpLoop(); // To get PerformDelayedNudge called.
1240   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1241   ASSERT_TRUE(scheduler()->IsBackingOff());
1242
1243   // Before we run the scheduled canary, trigger a server connection change.
1244   scheduler()->OnConnectionStatusChange();
1245   PumpLoop();
1246   connection()->SetServerReachable();
1247   connection()->UpdateConnectionStatus();
1248   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1249   base::MessageLoop::current()->RunUntilIdle();
1250 }
1251
1252 // Tests that we don't crash trying to run two canaries at once if we receive
1253 // extra connection status change notifications.  See crbug.com/190085.
1254 TEST_F(SyncSchedulerTest, DoubleCanaryInConfigure) {
1255   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
1256       .WillRepeatedly(DoAll(
1257               Invoke(sessions::test_util::SimulateConfigureConnectionFailure),
1258               Return(true)));
1259   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1260   connection()->SetServerNotReachable();
1261   connection()->UpdateConnectionStatus();
1262
1263   ModelTypeSet model_types(BOOKMARKS);
1264   CallbackCounter ready_counter;
1265   CallbackCounter retry_counter;
1266   ConfigurationParams params(
1267       GetUpdatesCallerInfo::RECONFIGURATION,
1268       model_types,
1269       TypesToRoutingInfo(model_types),
1270       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1271       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1272   scheduler()->ScheduleConfiguration(params);
1273
1274   scheduler()->OnConnectionStatusChange();
1275   scheduler()->OnConnectionStatusChange();
1276
1277   PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
1278 }
1279
1280 TEST_F(SyncSchedulerTest, PollFromCanaryAfterAuthError) {
1281   SyncShareTimes times;
1282   TimeDelta poll(TimeDelta::FromMilliseconds(15));
1283   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1284
1285   ::testing::InSequence seq;
1286   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1287       .WillRepeatedly(
1288           DoAll(Invoke(sessions::test_util::SimulatePollRetrySuccess),
1289                 RecordSyncShareMultiple(&times, kMinNumSamples)));
1290
1291   connection()->SetServerStatus(HttpResponse::SYNC_AUTH_ERROR);
1292   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1293
1294   // Run to wait for polling.
1295   RunLoop();
1296
1297   // Normally OnCredentialsUpdated calls TryCanaryJob that doesn't run Poll,
1298   // but after poll finished with auth error from poll timer it should retry
1299   // poll once more
1300   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1301       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollRetrySuccess),
1302                       RecordSyncShare(&times)));
1303   scheduler()->OnCredentialsUpdated();
1304   connection()->SetServerStatus(HttpResponse::SERVER_CONNECTION_OK);
1305   RunLoop();
1306   StopSyncScheduler();
1307 }
1308
1309 TEST_F(SyncSchedulerTest, SuccessfulRetry) {
1310   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1311
1312   SyncShareTimes times;
1313   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(1);
1314   scheduler()->OnReceivedGuRetryDelay(delay);
1315   EXPECT_EQ(delay, GetRetryTimerDelay());
1316
1317   EXPECT_CALL(*syncer(), RetrySyncShare(_,_))
1318       .WillOnce(
1319           DoAll(Invoke(sessions::test_util::SimulatePollRetrySuccess),
1320                 RecordSyncShare(&times)));
1321
1322   // Run to wait for retrying.
1323   RunLoop();
1324
1325   StopSyncScheduler();
1326 }
1327
1328 TEST_F(SyncSchedulerTest, FailedRetry) {
1329   UseMockDelayProvider();
1330   EXPECT_CALL(*delay(), GetDelay(_))
1331       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
1332
1333   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1334
1335   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(1);
1336   scheduler()->OnReceivedGuRetryDelay(delay);
1337
1338   EXPECT_CALL(*syncer(), RetrySyncShare(_,_))
1339       .WillOnce(
1340           DoAll(Invoke(sessions::test_util::SimulatePollRetryFailed),
1341                 QuitLoopNowAction()));
1342
1343   // Run to wait for retrying.
1344   RunLoop();
1345
1346   EXPECT_TRUE(scheduler()->IsBackingOff());
1347   EXPECT_CALL(*syncer(), RetrySyncShare(_,_))
1348       .WillOnce(
1349           DoAll(Invoke(sessions::test_util::SimulatePollRetrySuccess),
1350                 QuitLoopNowAction()));
1351
1352   // Run to wait for second retrying.
1353   RunLoop();
1354
1355   StopSyncScheduler();
1356 }
1357
1358 ACTION_P2(VerifyRetryTimerDelay, scheduler_test, expected_delay) {
1359   EXPECT_EQ(expected_delay, scheduler_test->GetRetryTimerDelay());
1360 }
1361
1362 TEST_F(SyncSchedulerTest, ReceiveNewRetryDelay) {
1363   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1364
1365   SyncShareTimes times;
1366   base::TimeDelta delay1 = base::TimeDelta::FromMilliseconds(100);
1367   base::TimeDelta delay2 = base::TimeDelta::FromMilliseconds(200);
1368
1369   scheduler()->ScheduleLocalRefreshRequest(zero(), ModelTypeSet(BOOKMARKS),
1370                                            FROM_HERE);
1371   scheduler()->OnReceivedGuRetryDelay(delay1);
1372
1373   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1374       .WillOnce(DoAll(
1375           WithoutArgs(VerifyRetryTimerDelay(this, delay1)),
1376           WithArg<2>(sessions::test_util::SimulateGuRetryDelayCommand(delay2)),
1377           WithoutArgs(VerifyRetryTimerDelay(this, delay2)),
1378           RecordSyncShare(&times)));
1379
1380   // Run nudge GU.
1381   RunLoop();
1382
1383   EXPECT_CALL(*syncer(), RetrySyncShare(_,_))
1384       .WillOnce(
1385           DoAll(Invoke(sessions::test_util::SimulatePollRetrySuccess),
1386                 RecordSyncShare(&times)));
1387
1388   // Run to wait for retrying.
1389   RunLoop();
1390
1391   StopSyncScheduler();
1392 }
1393
1394 }  // namespace syncer