sync/engine/sync_scheduler_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/bind.h"
   6 #include "base/callback.h"
   7 #include "base/compiler_specific.h"
   8 #include "base/memory/weak_ptr.h"
   9 #include "base/message_loop/message_loop.h"
  10 #include "base/test/test_timeouts.h"
  11 #include "sync/engine/backoff_delay_provider.h"
  12 #include "sync/engine/sync_scheduler_impl.h"
  13 #include "sync/engine/syncer.h"
  14 #include "sync/internal_api/public/base/cancelation_signal.h"
  15 #include "sync/internal_api/public/base/model_type_test_util.h"
  16 #include "sync/notifier/invalidation_util.h"
  17 #include "sync/notifier/object_id_invalidation_map.h"
  18 #include "sync/sessions/test_util.h"
  19 #include "sync/test/callback_counter.h"
  20 #include "sync/test/engine/fake_model_worker.h"
  21 #include "sync/test/engine/mock_connection_manager.h"
  22 #include "sync/test/engine/test_directory_setter_upper.h"
  23 #include "sync/util/extensions_activity.h"
  24 #include "testing/gmock/include/gmock/gmock.h"
  25 #include "testing/gtest/include/gtest/gtest.h"
  26
  27 using base::TimeDelta;
  28 using base::TimeTicks;
  29 using testing::_;
  30 using testing::AtLeast;
  31 using testing::DoAll;
  32 using testing::Invoke;
  33 using testing::Mock;
  34 using testing::Return;
  35 using testing::WithArg;
  36 using testing::WithArgs;
  37
  38 namespace syncer {
  39 using sessions::SyncSession;
  40 using sessions::SyncSessionContext;
  41 using sync_pb::GetUpdatesCallerInfo;
  42
  43 class MockSyncer : public Syncer {
  44  public:
  45   MockSyncer();
  46   MOCK_METHOD3(NormalSyncShare, bool(ModelTypeSet,
  47                                      const sessions::NudgeTracker&,
  48                                      sessions::SyncSession*));
  49   MOCK_METHOD3(ConfigureSyncShare,
  50                bool(ModelTypeSet,
  51                     sync_pb::GetUpdatesCallerInfo::GetUpdatesSource,
  52                     SyncSession*));
  53   MOCK_METHOD2(PollSyncShare, bool(ModelTypeSet, sessions::SyncSession*));
  54 };
  55
  56 MockSyncer::MockSyncer()
  57   : Syncer(NULL) {}
  58
  59 typedef std::vector<TimeTicks> SyncShareTimes;
  60
  61 void QuitLoopNow() {
  62   // We use QuitNow() instead of Quit() as the latter may get stalled
  63   // indefinitely in the presence of repeated timers with low delays
  64   // and a slow test (e.g., ThrottlingDoesThrottle [which has a poll
  65   // delay of 5ms] run under TSAN on the trybots).
  66   base::MessageLoop::current()->QuitNow();
  67 }
  68
  69 void RunLoop() {
  70   base::MessageLoop::current()->Run();
  71 }
  72
  73 void PumpLoop() {
  74   // Do it this way instead of RunAllPending to pump loop exactly once
  75   // (necessary in the presence of timers; see comment in
  76   // QuitLoopNow).
  77   base::MessageLoop::current()->PostTask(FROM_HERE, base::Bind(&QuitLoopNow));
  78   RunLoop();
  79 }
  80
  81 void PumpLoopFor(base::TimeDelta time) {
  82   // Allow the loop to run for the specified amount of time.
  83   base::MessageLoop::current()->PostDelayedTask(
  84       FROM_HERE, base::Bind(&QuitLoopNow), time);
  85   RunLoop();
  86 }
  87
  88 ModelSafeRoutingInfo TypesToRoutingInfo(ModelTypeSet types) {
  89   ModelSafeRoutingInfo routes;
  90   for (ModelTypeSet::Iterator iter = types.First(); iter.Good(); iter.Inc()) {
  91     routes[iter.Get()] = GROUP_PASSIVE;
  92   }
  93   return routes;
  94 }
  95
  96 // Convenient to use in tests wishing to analyze SyncShare calls over time.
  97 static const size_t kMinNumSamples = 5;
  98 class SyncSchedulerTest : public testing::Test {
  99  public:
 100   SyncSchedulerTest() : syncer_(NULL), delay_(NULL), weak_ptr_factory_(this) {}
 101
 102   class MockDelayProvider : public BackoffDelayProvider {
 103    public:
 104     MockDelayProvider() : BackoffDelayProvider(
 105         TimeDelta::FromSeconds(kInitialBackoffRetrySeconds),
 106         TimeDelta::FromSeconds(kInitialBackoffImmediateRetrySeconds)) {
 107     }
 108
 109     MOCK_METHOD1(GetDelay, TimeDelta(const TimeDelta&));
 110   };
 111
 112   virtual void SetUp() {
 113     dir_maker_.SetUp();
 114     syncer_ = new testing::StrictMock<MockSyncer>();
 115     delay_ = NULL;
 116     extensions_activity_ = new ExtensionsActivity();
 117
 118     routing_info_[BOOKMARKS] = GROUP_UI;
 119     routing_info_[AUTOFILL] = GROUP_DB;
 120     routing_info_[THEMES] = GROUP_UI;
 121     routing_info_[NIGORI] = GROUP_PASSIVE;
 122
 123     workers_.clear();
 124     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_UI)));
 125     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_DB)));
 126     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_PASSIVE)));
 127
 128     std::vector<ModelSafeWorker*> workers;
 129     for (std::vector<scoped_refptr<FakeModelWorker> >::iterator it =
 130          workers_.begin(); it != workers_.end(); ++it) {
 131       workers.push_back(it->get());
 132     }
 133
 134     connection_.reset(new MockConnectionManager(directory(),
 135                                                 &cancelation_signal_));
 136     connection_->SetServerReachable();
 137     context_.reset(new SyncSessionContext(
 138             connection_.get(), directory(), workers,
 139             extensions_activity_.get(),
 140             std::vector<SyncEngineEventListener*>(), NULL, NULL,
 141             true,  // enable keystore encryption
 142             false,  // force enable pre-commit GU avoidance
 143             "fake_invalidator_client_id"));
 144     context_->set_routing_info(routing_info_);
 145     context_->set_notifications_enabled(true);
 146     context_->set_account_name("Test");
 147     scheduler_.reset(
 148         new SyncSchedulerImpl("TestSyncScheduler",
 149             BackoffDelayProvider::FromDefaults(),
 150             context(),
 151             syncer_));
 152   }
 153
 154   SyncSchedulerImpl* scheduler() { return scheduler_.get(); }
 155   const ModelSafeRoutingInfo& routing_info() { return routing_info_; }
 156   MockSyncer* syncer() { return syncer_; }
 157   MockDelayProvider* delay() { return delay_; }
 158   MockConnectionManager* connection() { return connection_.get(); }
 159   TimeDelta zero() { return TimeDelta::FromSeconds(0); }
 160   TimeDelta timeout() {
 161     return TestTimeouts::action_timeout();
 162   }
 163
 164   virtual void TearDown() {
 165     PumpLoop();
 166     scheduler_.reset();
 167     PumpLoop();
 168     dir_maker_.TearDown();
 169   }
 170
 171   void AnalyzePollRun(const SyncShareTimes& times, size_t min_num_samples,
 172       const TimeTicks& optimal_start, const TimeDelta& poll_interval) {
 173     EXPECT_GE(times.size(), min_num_samples);
 174     for (size_t i = 0; i < times.size(); i++) {
 175       SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
 176       TimeTicks optimal_next_sync = optimal_start + poll_interval * i;
 177       EXPECT_GE(times[i], optimal_next_sync);
 178     }
 179   }
 180
 181   void DoQuitLoopNow() {
 182     QuitLoopNow();
 183   }
 184
 185   void StartSyncScheduler(SyncScheduler::Mode mode) {
 186     scheduler()->Start(mode);
 187   }
 188
 189   // This stops the scheduler synchronously.
 190   void StopSyncScheduler() {
 191     base::MessageLoop::current()->PostTask(
 192         FROM_HERE,
 193         base::Bind(&SyncSchedulerTest::DoQuitLoopNow,
 194                    weak_ptr_factory_.GetWeakPtr()));
 195     RunLoop();
 196   }
 197
 198   bool RunAndGetBackoff() {
 199     ModelTypeSet nudge_types(BOOKMARKS);
 200     StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 201
 202     scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
 203     RunLoop();
 204
 205     return scheduler()->IsBackingOff();
 206   }
 207
 208   void UseMockDelayProvider() {
 209     delay_ = new MockDelayProvider();
 210     scheduler_->delay_provider_.reset(delay_);
 211   }
 212
 213   SyncSessionContext* context() { return context_.get(); }
 214
 215   ModelTypeSet GetThrottledTypes() {
 216     return scheduler_->nudge_tracker_.GetThrottledTypes();
 217   }
 218
 219  private:
 220   syncable::Directory* directory() {
 221     return dir_maker_.directory();
 222   }
 223
 224   base::MessageLoop loop_;
 225   TestDirectorySetterUpper dir_maker_;
 226   CancelationSignal cancelation_signal_;
 227   scoped_ptr<MockConnectionManager> connection_;
 228   scoped_ptr<SyncSessionContext> context_;
 229   scoped_ptr<SyncSchedulerImpl> scheduler_;
 230   MockSyncer* syncer_;
 231   MockDelayProvider* delay_;
 232   std::vector<scoped_refptr<FakeModelWorker> > workers_;
 233   scoped_refptr<ExtensionsActivity> extensions_activity_;
 234   ModelSafeRoutingInfo routing_info_;
 235   base::WeakPtrFactory<SyncSchedulerTest> weak_ptr_factory_;
 236 };
 237
 238 void RecordSyncShareImpl(SyncShareTimes* times) {
 239   times->push_back(TimeTicks::Now());
 240 }
 241
 242 ACTION_P(RecordSyncShare, times) {
 243   RecordSyncShareImpl(times);
 244   if (base::MessageLoop::current()->is_running())
 245     QuitLoopNow();
 246   return true;
 247 }
 248
 249 ACTION_P2(RecordSyncShareMultiple, times, quit_after) {
 250   RecordSyncShareImpl(times);
 251   EXPECT_LE(times->size(), quit_after);
 252   if (times->size() >= quit_after &&
 253       base::MessageLoop::current()->is_running()) {
 254     QuitLoopNow();
 255   }
 256   return true;
 257 }
 258
 259 ACTION(AddFailureAndQuitLoopNow) {
 260   ADD_FAILURE();
 261   QuitLoopNow();
 262   return true;
 263 }
 264
 265 ACTION(QuitLoopNowAction) {
 266   QuitLoopNow();
 267   return true;
 268 }
 269
 270 // Test nudge scheduling.
 271 TEST_F(SyncSchedulerTest, Nudge) {
 272   SyncShareTimes times;
 273   ModelTypeSet model_types(BOOKMARKS);
 274
 275   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 276       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 277                       RecordSyncShare(&times)))
 278       .RetiresOnSaturation();
 279
 280   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 281
 282   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 283   RunLoop();
 284
 285   Mock::VerifyAndClearExpectations(syncer());
 286
 287   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 288   SyncShareTimes times2;
 289   model_types.Remove(BOOKMARKS);
 290   model_types.Put(AUTOFILL);
 291   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 292       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 293                       RecordSyncShare(&times2)));
 294   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 295   RunLoop();
 296 }
 297
 298 // Make sure a regular config command is scheduled fine in the absence of any
 299 // errors.
 300 TEST_F(SyncSchedulerTest, Config) {
 301   SyncShareTimes times;
 302   const ModelTypeSet model_types(BOOKMARKS);
 303
 304   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 305       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 306                       RecordSyncShare(&times)));
 307
 308   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 309
 310   CallbackCounter counter;
 311   ConfigurationParams params(
 312       GetUpdatesCallerInfo::RECONFIGURATION,
 313       model_types,
 314       TypesToRoutingInfo(model_types),
 315       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
 316   ASSERT_TRUE(scheduler()->ScheduleConfiguration(params));
 317   ASSERT_EQ(1, counter.times_called());
 318 }
 319
 320 // Simulate a failure and make sure the config request is retried.
 321 TEST_F(SyncSchedulerTest, ConfigWithBackingOff) {
 322   UseMockDelayProvider();
 323   EXPECT_CALL(*delay(), GetDelay(_))
 324       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
 325   SyncShareTimes times;
 326   const ModelTypeSet model_types(BOOKMARKS);
 327
 328   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 329
 330   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 331       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 332                       RecordSyncShare(&times)));
 333
 334   CallbackCounter counter;
 335   ConfigurationParams params(
 336       GetUpdatesCallerInfo::RECONFIGURATION,
 337       model_types,
 338       TypesToRoutingInfo(model_types),
 339       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
 340   ASSERT_FALSE(scheduler()->ScheduleConfiguration(params));
 341   ASSERT_EQ(0, counter.times_called());
 342
 343   Mock::VerifyAndClearExpectations(syncer());
 344
 345   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 346       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 347                       RecordSyncShare(&times)));
 348   RunLoop();
 349
 350   ASSERT_EQ(1, counter.times_called());
 351 }
 352
 353 // Issue a nudge when the config has failed. Make sure both the config and
 354 // nudge are executed.
 355 TEST_F(SyncSchedulerTest, NudgeWithConfigWithBackingOff) {
 356   const ModelTypeSet model_types(BOOKMARKS);
 357   UseMockDelayProvider();
 358   EXPECT_CALL(*delay(), GetDelay(_))
 359       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(50)));
 360   SyncShareTimes times;
 361
 362   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 363
 364   // Request a configure and make sure it fails.
 365   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 366       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 367                       RecordSyncShare(&times)));
 368   CallbackCounter counter;
 369   ConfigurationParams params(
 370       GetUpdatesCallerInfo::RECONFIGURATION,
 371       model_types,
 372       TypesToRoutingInfo(model_types),
 373       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
 374   ASSERT_FALSE(scheduler()->ScheduleConfiguration(params));
 375   ASSERT_EQ(0, counter.times_called());
 376   Mock::VerifyAndClearExpectations(syncer());
 377
 378   // Ask for a nudge while dealing with repeated configure failure.
 379   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 380       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 381                       RecordSyncShare(&times)));
 382   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 383   RunLoop();
 384   // Note that we're not RunLoop()ing for the NUDGE we just scheduled, but
 385   // for the first retry attempt from the config job (after
 386   // waiting ~+/- 50ms).
 387   Mock::VerifyAndClearExpectations(syncer());
 388   ASSERT_EQ(0, counter.times_called());
 389
 390   // Let the next configure retry succeed.
 391   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 392       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 393                       RecordSyncShare(&times)));
 394   RunLoop();
 395
 396   // Now change the mode so nudge can execute.
 397   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 398       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 399                       RecordSyncShare(&times)));
 400   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 401 }
 402
 403 // Test that nudges are coalesced.
 404 TEST_F(SyncSchedulerTest, NudgeCoalescing) {
 405   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 406
 407   SyncShareTimes times;
 408   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 409       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 410                       RecordSyncShare(&times)));
 411   const ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3(THEMES);
 412   TimeDelta delay = zero();
 413   TimeTicks optimal_time = TimeTicks::Now() + delay;
 414   scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
 415   scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
 416   RunLoop();
 417
 418   ASSERT_EQ(1U, times.size());
 419   EXPECT_GE(times[0], optimal_time);
 420
 421   Mock::VerifyAndClearExpectations(syncer());
 422
 423   SyncShareTimes times2;
 424   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 425       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 426                       RecordSyncShare(&times2)));
 427   scheduler()->ScheduleLocalNudge(zero(), types3, FROM_HERE);
 428   RunLoop();
 429 }
 430
 431 // Test that nudges are coalesced.
 432 TEST_F(SyncSchedulerTest, NudgeCoalescingWithDifferentTimings) {
 433   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 434
 435   SyncShareTimes times;
 436   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 437       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 438                       RecordSyncShare(&times)));
 439   ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3;
 440
 441   // Create a huge time delay.
 442   TimeDelta delay = TimeDelta::FromDays(1);
 443
 444   scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
 445   scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
 446
 447   TimeTicks min_time = TimeTicks::Now();
 448   TimeTicks max_time = TimeTicks::Now() + delay;
 449
 450   RunLoop();
 451   Mock::VerifyAndClearExpectations(syncer());
 452
 453   // Make sure the sync happened at the right time.
 454   ASSERT_EQ(1U, times.size());
 455   EXPECT_GE(times[0], min_time);
 456   EXPECT_LE(times[0], max_time);
 457 }
 458
 459 // Test nudge scheduling.
 460 TEST_F(SyncSchedulerTest, NudgeWithStates) {
 461   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 462
 463   SyncShareTimes times1;
 464   ObjectIdInvalidationMap invalidations1 =
 465       BuildInvalidationMap(BOOKMARKS, 10, "test");
 466   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 467       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 468                       RecordSyncShare(&times1)))
 469       .RetiresOnSaturation();
 470   scheduler()->ScheduleInvalidationNudge(zero(), invalidations1, FROM_HERE);
 471   RunLoop();
 472
 473   Mock::VerifyAndClearExpectations(syncer());
 474
 475   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 476   SyncShareTimes times2;
 477   ObjectIdInvalidationMap invalidations2 =
 478       BuildInvalidationMap(AUTOFILL, 10, "test2");
 479   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 480       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 481                       RecordSyncShare(&times2)));
 482   scheduler()->ScheduleInvalidationNudge(zero(), invalidations2, FROM_HERE);
 483   RunLoop();
 484 }
 485
 486 // Test that polling works as expected.
 487 TEST_F(SyncSchedulerTest, Polling) {
 488   SyncShareTimes times;
 489   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 490   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 491       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 492            RecordSyncShareMultiple(&times, kMinNumSamples)));
 493
 494   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
 495
 496   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 497   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 498
 499   // Run again to wait for polling.
 500   RunLoop();
 501
 502   StopSyncScheduler();
 503   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 504 }
 505
 506 // Test that the short poll interval is used.
 507 TEST_F(SyncSchedulerTest, PollNotificationsDisabled) {
 508   SyncShareTimes times;
 509   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 510   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 511       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 512            RecordSyncShareMultiple(&times, kMinNumSamples)));
 513
 514   scheduler()->OnReceivedShortPollIntervalUpdate(poll_interval);
 515   scheduler()->SetNotificationsEnabled(false);
 516
 517   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 518   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 519
 520   // Run again to wait for polling.
 521   RunLoop();
 522
 523   StopSyncScheduler();
 524   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 525 }
 526
 527 // Test that polling intervals are updated when needed.
 528 TEST_F(SyncSchedulerTest, PollIntervalUpdate) {
 529   SyncShareTimes times;
 530   TimeDelta poll1(TimeDelta::FromMilliseconds(120));
 531   TimeDelta poll2(TimeDelta::FromMilliseconds(30));
 532   scheduler()->OnReceivedLongPollIntervalUpdate(poll1);
 533   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 534       .WillOnce(DoAll(
 535           WithArgs<0,1>(
 536               sessions::test_util::SimulatePollIntervalUpdate(poll2)),
 537           Return(true)))
 538       .WillRepeatedly(
 539           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 540                 WithArg<1>(
 541                     RecordSyncShareMultiple(&times, kMinNumSamples))));
 542
 543   TimeTicks optimal_start = TimeTicks::Now() + poll1 + poll2;
 544   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 545
 546   // Run again to wait for polling.
 547   RunLoop();
 548
 549   StopSyncScheduler();
 550   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll2);
 551 }
 552
 553 // Test that the sessions commit delay is updated when needed.
 554 TEST_F(SyncSchedulerTest, SessionsCommitDelay) {
 555   SyncShareTimes times;
 556   TimeDelta delay1(TimeDelta::FromMilliseconds(120));
 557   TimeDelta delay2(TimeDelta::FromMilliseconds(30));
 558   scheduler()->OnReceivedSessionsCommitDelay(delay1);
 559
 560   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 561       .WillOnce(
 562           DoAll(
 563               WithArgs<0,1,2>(
 564                   sessions::test_util::SimulateSessionsCommitDelayUpdate(
 565                       delay2)),
 566               Invoke(sessions::test_util::SimulateNormalSuccess),
 567               QuitLoopNowAction()));
 568
 569   EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
 570   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 571
 572   EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
 573   const ModelTypeSet model_types(BOOKMARKS);
 574   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 575   RunLoop();
 576
 577   EXPECT_EQ(delay2, scheduler()->GetSessionsCommitDelay());
 578   StopSyncScheduler();
 579 }
 580
 581 // Test that no syncing occurs when throttled.
 582 TEST_F(SyncSchedulerTest, ThrottlingDoesThrottle) {
 583   const ModelTypeSet types(BOOKMARKS);
 584   TimeDelta poll(TimeDelta::FromMilliseconds(5));
 585   TimeDelta throttle(TimeDelta::FromMinutes(10));
 586   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 587
 588   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 589       .WillOnce(DoAll(
 590           WithArg<2>(sessions::test_util::SimulateThrottled(throttle)),
 591           Return(true)))
 592       .WillRepeatedly(AddFailureAndQuitLoopNow());
 593
 594   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 595
 596   scheduler()->ScheduleLocalNudge(
 597       TimeDelta::FromMicroseconds(1), types, FROM_HERE);
 598   PumpLoop();
 599
 600   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 601
 602   CallbackCounter counter;
 603   ConfigurationParams params(
 604       GetUpdatesCallerInfo::RECONFIGURATION,
 605       types,
 606       TypesToRoutingInfo(types),
 607       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
 608   ASSERT_FALSE(scheduler()->ScheduleConfiguration(params));
 609   ASSERT_EQ(0, counter.times_called());
 610 }
 611
 612 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromPoll) {
 613   SyncShareTimes times;
 614   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 615   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 616   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 617
 618   ::testing::InSequence seq;
 619   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 620       .WillOnce(DoAll(
 621           WithArg<1>(sessions::test_util::SimulateThrottled(throttle1)),
 622           Return(true)))
 623       .RetiresOnSaturation();
 624   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 625       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 626            RecordSyncShareMultiple(&times, kMinNumSamples)));
 627
 628   TimeTicks optimal_start = TimeTicks::Now() + poll + throttle1;
 629   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 630
 631   // Run again to wait for polling.
 632   RunLoop();
 633
 634   StopSyncScheduler();
 635   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll);
 636 }
 637
 638 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromNudge) {
 639   SyncShareTimes times;
 640   TimeDelta poll(TimeDelta::FromDays(1));
 641   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 642   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 643
 644   ::testing::InSequence seq;
 645   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 646       .WillOnce(DoAll(
 647           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 648           Return(true)))
 649       .RetiresOnSaturation();
 650   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 651       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 652                       QuitLoopNowAction()));
 653
 654   const ModelTypeSet types(BOOKMARKS);
 655   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 656   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 657
 658   PumpLoop();
 659   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 660   RunLoop();
 661   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 662
 663   StopSyncScheduler();
 664 }
 665
 666 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromConfigure) {
 667   SyncShareTimes times;
 668   TimeDelta poll(TimeDelta::FromDays(1));
 669   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 670   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 671
 672   ::testing::InSequence seq;
 673   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 674       .WillOnce(DoAll(
 675           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 676           Return(true)))
 677       .RetiresOnSaturation();
 678   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 679       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 680                       QuitLoopNowAction()));
 681
 682   const ModelTypeSet types(BOOKMARKS);
 683   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 684
 685   CallbackCounter counter;
 686   ConfigurationParams params(
 687       GetUpdatesCallerInfo::RECONFIGURATION,
 688       types,
 689       TypesToRoutingInfo(types),
 690       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
 691   EXPECT_FALSE(scheduler()->ScheduleConfiguration(params));
 692   EXPECT_EQ(0, counter.times_called());
 693   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 694
 695   RunLoop();
 696   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 697
 698   StopSyncScheduler();
 699 }
 700
 701 TEST_F(SyncSchedulerTest, TypeThrottlingBlocksNudge) {
 702   UseMockDelayProvider();
 703   EXPECT_CALL(*delay(), GetDelay(_))
 704       .WillRepeatedly(Return(zero()));
 705
 706   TimeDelta poll(TimeDelta::FromDays(1));
 707   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 708   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 709
 710   const ModelTypeSet types(BOOKMARKS);
 711
 712   ::testing::InSequence seq;
 713   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 714       .WillOnce(DoAll(
 715           WithArg<2>(
 716               sessions::test_util::SimulateTypesThrottled(types, throttle1)),
 717           Return(true)))
 718       .RetiresOnSaturation();
 719
 720   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 721   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 722   PumpLoop();
 723   EXPECT_TRUE(GetThrottledTypes().HasAll(types));
 724
 725   // This won't cause a sync cycle because the types are throttled.
 726   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 727   PumpLoop();
 728
 729   StopSyncScheduler();
 730 }
 731
 732 TEST_F(SyncSchedulerTest, TypeThrottlingDoesBlockOtherSources) {
 733   UseMockDelayProvider();
 734   EXPECT_CALL(*delay(), GetDelay(_))
 735       .WillRepeatedly(Return(zero()));
 736
 737   SyncShareTimes times;
 738   TimeDelta poll(TimeDelta::FromDays(1));
 739   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 740   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 741
 742   const ModelTypeSet throttled_types(BOOKMARKS);
 743   const ModelTypeSet unthrottled_types(PREFERENCES);
 744
 745   ::testing::InSequence seq;
 746   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 747       .WillOnce(DoAll(
 748           WithArg<2>(
 749               sessions::test_util::SimulateTypesThrottled(
 750                   throttled_types, throttle1)),
 751           Return(true)))
 752       .RetiresOnSaturation();
 753
 754   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 755   scheduler()->ScheduleLocalNudge(zero(), throttled_types, FROM_HERE);
 756   PumpLoop();
 757   EXPECT_TRUE(GetThrottledTypes().HasAll(throttled_types));
 758
 759   // Ignore invalidations for throttled types.
 760   ObjectIdInvalidationMap invalidations =
 761       BuildInvalidationMap(BOOKMARKS, 10, "test");
 762   scheduler()->ScheduleInvalidationNudge(zero(), invalidations, FROM_HERE);
 763   PumpLoop();
 764
 765   // Ignore refresh requests for throttled types.
 766   scheduler()->ScheduleLocalRefreshRequest(zero(), throttled_types, FROM_HERE);
 767   PumpLoop();
 768
 769   Mock::VerifyAndClearExpectations(syncer());
 770
 771   // Local nudges for non-throttled types will trigger a sync.
 772   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 773       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 774                             RecordSyncShare(&times)));
 775   scheduler()->ScheduleLocalNudge(zero(), unthrottled_types, FROM_HERE);
 776   RunLoop();
 777   Mock::VerifyAndClearExpectations(syncer());
 778
 779   StopSyncScheduler();
 780 }
 781
 782 // Test nudges / polls don't run in config mode and config tasks do.
 783 TEST_F(SyncSchedulerTest, ConfigurationMode) {
 784   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 785   SyncShareTimes times;
 786   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 787
 788   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 789
 790   const ModelTypeSet nudge_types(AUTOFILL);
 791   scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
 792   scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
 793
 794   const ModelTypeSet config_types(BOOKMARKS);
 795
 796   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 797       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 798                       RecordSyncShare(&times)))
 799       .RetiresOnSaturation();
 800   CallbackCounter counter;
 801   ConfigurationParams params(
 802       GetUpdatesCallerInfo::RECONFIGURATION,
 803       config_types,
 804       TypesToRoutingInfo(config_types),
 805       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
 806   ASSERT_TRUE(scheduler()->ScheduleConfiguration(params));
 807   ASSERT_EQ(1, counter.times_called());
 808   Mock::VerifyAndClearExpectations(syncer());
 809
 810   // Switch to NORMAL_MODE to ensure NUDGES were properly saved and run.
 811   scheduler()->OnReceivedLongPollIntervalUpdate(TimeDelta::FromDays(1));
 812   SyncShareTimes times2;
 813   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 814       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 815                       RecordSyncShare(&times2)));
 816
 817   // TODO(tim): Figure out how to remove this dangerous need to reset
 818   // routing info between mode switches.
 819   context()->set_routing_info(routing_info());
 820   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 821
 822   PumpLoop();
 823 }
 824
 825 class BackoffTriggersSyncSchedulerTest : public SyncSchedulerTest {
 826   virtual void SetUp() {
 827     SyncSchedulerTest::SetUp();
 828     UseMockDelayProvider();
 829     EXPECT_CALL(*delay(), GetDelay(_))
 830         .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
 831   }
 832
 833   virtual void TearDown() {
 834     StopSyncScheduler();
 835     SyncSchedulerTest::TearDown();
 836   }
 837 };
 838
 839 // Have the sycner fail during commit.  Expect that the scheduler enters
 840 // backoff.
 841 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnce) {
 842   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 843       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
 844                       QuitLoopNowAction()));
 845   EXPECT_TRUE(RunAndGetBackoff());
 846 }
 847
 848 // Have the syncer fail during download updates and succeed on the first
 849 // retry.  Expect that this clears the backoff state.
 850 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadOnceThenSucceed) {
 851   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 852       .WillOnce(DoAll(
 853           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 854           Return(true)))
 855       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 856                       QuitLoopNowAction()));
 857   EXPECT_FALSE(RunAndGetBackoff());
 858 }
 859
 860 // Have the syncer fail during commit and succeed on the first retry.  Expect
 861 // that this clears the backoff state.
 862 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnceThenSucceed) {
 863   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 864       .WillOnce(DoAll(
 865           Invoke(sessions::test_util::SimulateCommitFailed),
 866           Return(true)))
 867       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 868                       QuitLoopNowAction()));
 869   EXPECT_FALSE(RunAndGetBackoff());
 870 }
 871
 872 // Have the syncer fail to download updates and fail again on the retry.
 873 // Expect this will leave the scheduler in backoff.
 874 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadTwice) {
 875   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 876       .WillOnce(DoAll(
 877           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 878           Return(true)))
 879       .WillRepeatedly(DoAll(
 880               Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 881               QuitLoopNowAction()));
 882   EXPECT_TRUE(RunAndGetBackoff());
 883 }
 884
 885 // Have the syncer fail to get the encryption key yet succeed in downloading
 886 // updates. Expect this will leave the scheduler in backoff.
 887 TEST_F(BackoffTriggersSyncSchedulerTest, FailGetEncryptionKey) {
 888   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 889       .WillOnce(DoAll(
 890           Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
 891           Return(true)))
 892       .WillRepeatedly(DoAll(
 893               Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
 894               QuitLoopNowAction()));
 895   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 896
 897   ModelTypeSet types(BOOKMARKS);
 898   CallbackCounter counter;
 899   ConfigurationParams params(
 900       GetUpdatesCallerInfo::RECONFIGURATION,
 901       types,
 902       TypesToRoutingInfo(types),
 903       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
 904   scheduler()->ScheduleConfiguration(params);
 905   RunLoop();
 906
 907   EXPECT_TRUE(scheduler()->IsBackingOff());
 908 }
 909
 910 // Test that no polls or extraneous nudges occur when in backoff.
 911 TEST_F(SyncSchedulerTest, BackoffDropsJobs) {
 912   SyncShareTimes times;
 913   TimeDelta poll(TimeDelta::FromMilliseconds(5));
 914   const ModelTypeSet types(BOOKMARKS);
 915   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 916   UseMockDelayProvider();
 917
 918   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 919       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
 920                       RecordSyncShareMultiple(&times, 1U)));
 921   EXPECT_CALL(*delay(), GetDelay(_)).
 922       WillRepeatedly(Return(TimeDelta::FromDays(1)));
 923
 924   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 925
 926   // This nudge should fail and put us into backoff.  Thanks to our mock
 927   // GetDelay() setup above, this will be a long backoff.
 928   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 929   RunLoop();
 930
 931   // From this point forward, no SyncShare functions should be invoked.
 932   Mock::VerifyAndClearExpectations(syncer());
 933
 934   // Wait a while (10x poll interval) so a few poll jobs will be attempted.
 935   PumpLoopFor(poll * 10);
 936
 937   // Try (and fail) to schedule a nudge.
 938   scheduler()->ScheduleLocalNudge(
 939       base::TimeDelta::FromMilliseconds(1),
 940       types,
 941       FROM_HERE);
 942
 943   Mock::VerifyAndClearExpectations(syncer());
 944   Mock::VerifyAndClearExpectations(delay());
 945
 946   EXPECT_CALL(*delay(), GetDelay(_)).Times(0);
 947
 948   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 949
 950   CallbackCounter counter;
 951   ConfigurationParams params(
 952       GetUpdatesCallerInfo::RECONFIGURATION,
 953       types,
 954       TypesToRoutingInfo(types),
 955       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
 956   ASSERT_FALSE(scheduler()->ScheduleConfiguration(params));
 957   ASSERT_EQ(0, counter.times_called());
 958 }
 959
 960 // Test that backoff is shaping traffic properly with consecutive errors.
 961 TEST_F(SyncSchedulerTest, BackoffElevation) {
 962   SyncShareTimes times;
 963   UseMockDelayProvider();
 964
 965   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_)).Times(kMinNumSamples)
 966       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
 967           RecordSyncShareMultiple(&times, kMinNumSamples)));
 968
 969   const TimeDelta first = TimeDelta::FromSeconds(kInitialBackoffRetrySeconds);
 970   const TimeDelta second = TimeDelta::FromMilliseconds(2);
 971   const TimeDelta third = TimeDelta::FromMilliseconds(3);
 972   const TimeDelta fourth = TimeDelta::FromMilliseconds(4);
 973   const TimeDelta fifth = TimeDelta::FromMilliseconds(5);
 974   const TimeDelta sixth = TimeDelta::FromDays(1);
 975
 976   EXPECT_CALL(*delay(), GetDelay(first)).WillOnce(Return(second))
 977           .RetiresOnSaturation();
 978   EXPECT_CALL(*delay(), GetDelay(second)).WillOnce(Return(third))
 979           .RetiresOnSaturation();
 980   EXPECT_CALL(*delay(), GetDelay(third)).WillOnce(Return(fourth))
 981           .RetiresOnSaturation();
 982   EXPECT_CALL(*delay(), GetDelay(fourth)).WillOnce(Return(fifth))
 983           .RetiresOnSaturation();
 984   EXPECT_CALL(*delay(), GetDelay(fifth)).WillOnce(Return(sixth));
 985
 986   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 987
 988   // Run again with a nudge.
 989   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
 990   RunLoop();
 991
 992   ASSERT_EQ(kMinNumSamples, times.size());
 993   EXPECT_GE(times[1] - times[0], second);
 994   EXPECT_GE(times[2] - times[1], third);
 995   EXPECT_GE(times[3] - times[2], fourth);
 996   EXPECT_GE(times[4] - times[3], fifth);
 997 }
 998
 999 // Test that things go back to normal once a retry makes forward progress.
1000 TEST_F(SyncSchedulerTest, BackoffRelief) {
1001   SyncShareTimes times;
1002   const TimeDelta poll(TimeDelta::FromMilliseconds(10));
1003   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1004   UseMockDelayProvider();
1005
1006   const TimeDelta backoff = TimeDelta::FromMilliseconds(5);
1007   EXPECT_CALL(*delay(), GetDelay(_)).WillOnce(Return(backoff));
1008
1009   // Optimal start for the post-backoff poll party.
1010   TimeTicks optimal_start = TimeTicks::Now();
1011   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1012
1013   // Kick off the test with a failed nudge.
1014   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1015       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1016                       RecordSyncShare(&times)));
1017   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1018   RunLoop();
1019   Mock::VerifyAndClearExpectations(syncer());
1020   TimeTicks optimal_job_time = optimal_start;
1021   ASSERT_EQ(1U, times.size());
1022   EXPECT_GE(times[0], optimal_job_time);
1023
1024   // The retry succeeds.
1025   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1026       .WillOnce(DoAll(
1027               Invoke(sessions::test_util::SimulateNormalSuccess),
1028               RecordSyncShare(&times)));
1029   RunLoop();
1030   Mock::VerifyAndClearExpectations(syncer());
1031   optimal_job_time = optimal_job_time + backoff;
1032   ASSERT_EQ(2U, times.size());
1033   EXPECT_GE(times[1], optimal_job_time);
1034
1035   // Now let the Poll timer do its thing.
1036   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1037       .WillRepeatedly(DoAll(
1038               Invoke(sessions::test_util::SimulatePollSuccess),
1039               RecordSyncShareMultiple(&times, kMinNumSamples)));
1040   RunLoop();
1041   Mock::VerifyAndClearExpectations(syncer());
1042   ASSERT_EQ(kMinNumSamples, times.size());
1043   for (size_t i = 2; i < times.size(); i++) {
1044     optimal_job_time = optimal_job_time + poll;
1045     SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
1046     EXPECT_GE(times[i], optimal_job_time);
1047   }
1048
1049   StopSyncScheduler();
1050 }
1051
1052 // Test that poll failures are ignored.  They should have no effect on
1053 // subsequent poll attempts, nor should they trigger a backoff/retry.
1054 TEST_F(SyncSchedulerTest, TransientPollFailure) {
1055   SyncShareTimes times;
1056   const TimeDelta poll_interval(TimeDelta::FromMilliseconds(1));
1057   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
1058   UseMockDelayProvider(); // Will cause test failure if backoff is initiated.
1059
1060   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1061       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollFailed),
1062                       RecordSyncShare(&times)))
1063       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1064                       RecordSyncShare(&times)));
1065
1066   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1067
1068   // Run the unsucessful poll. The failed poll should not trigger backoff.
1069   RunLoop();
1070   EXPECT_FALSE(scheduler()->IsBackingOff());
1071
1072   // Run the successful poll.
1073   RunLoop();
1074   EXPECT_FALSE(scheduler()->IsBackingOff());
1075 }
1076
1077 // Test that starting the syncer thread without a valid connection doesn't
1078 // break things when a connection is detected.
1079 TEST_F(SyncSchedulerTest, StartWhenNotConnected) {
1080   connection()->SetServerNotReachable();
1081   connection()->UpdateConnectionStatus();
1082   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1083     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1084                     Return(true)))
1085     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1086                     Return(true)));
1087   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1088
1089   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1090   // Should save the nudge for until after the server is reachable.
1091   base::MessageLoop::current()->RunUntilIdle();
1092
1093   scheduler()->OnConnectionStatusChange();
1094   connection()->SetServerReachable();
1095   connection()->UpdateConnectionStatus();
1096   base::MessageLoop::current()->RunUntilIdle();
1097 }
1098
1099 TEST_F(SyncSchedulerTest, ServerConnectionChangeDuringBackoff) {
1100   UseMockDelayProvider();
1101   EXPECT_CALL(*delay(), GetDelay(_))
1102       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1103
1104   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1105   connection()->SetServerNotReachable();
1106   connection()->UpdateConnectionStatus();
1107
1108   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1109     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1110                     Return(true)))
1111     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1112                     Return(true)));
1113
1114   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1115
1116   PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
1117   ASSERT_TRUE(scheduler()->IsBackingOff());
1118
1119   // Before we run the scheduled canary, trigger a server connection change.
1120   scheduler()->OnConnectionStatusChange();
1121   connection()->SetServerReachable();
1122   connection()->UpdateConnectionStatus();
1123   base::MessageLoop::current()->RunUntilIdle();
1124 }
1125
1126 // This was supposed to test the scenario where we receive a nudge while a
1127 // connection change canary is scheduled, but has not run yet.  Since we've made
1128 // the connection change canary synchronous, this is no longer possible.
1129 TEST_F(SyncSchedulerTest, ConnectionChangeCanaryPreemptedByNudge) {
1130   UseMockDelayProvider();
1131   EXPECT_CALL(*delay(), GetDelay(_))
1132       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1133
1134   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1135   connection()->SetServerNotReachable();
1136   connection()->UpdateConnectionStatus();
1137
1138   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1139     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1140                     Return(true)))
1141     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1142                     Return(true)))
1143     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1144                     QuitLoopNowAction()));
1145
1146   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1147
1148   PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
1149   ASSERT_TRUE(scheduler()->IsBackingOff());
1150
1151   // Before we run the scheduled canary, trigger a server connection change.
1152   scheduler()->OnConnectionStatusChange();
1153   connection()->SetServerReachable();
1154   connection()->UpdateConnectionStatus();
1155   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1156   base::MessageLoop::current()->RunUntilIdle();
1157 }
1158
1159 // Tests that we don't crash trying to run two canaries at once if we receive
1160 // extra connection status change notifications.  See crbug.com/190085.
1161 TEST_F(SyncSchedulerTest, DoubleCanaryInConfigure) {
1162   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
1163       .WillRepeatedly(DoAll(
1164               Invoke(sessions::test_util::SimulateConfigureConnectionFailure),
1165               Return(true)));
1166   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1167   connection()->SetServerNotReachable();
1168   connection()->UpdateConnectionStatus();
1169
1170   ModelTypeSet model_types(BOOKMARKS);
1171   CallbackCounter counter;
1172   ConfigurationParams params(
1173       GetUpdatesCallerInfo::RECONFIGURATION,
1174       model_types,
1175       TypesToRoutingInfo(model_types),
1176       base::Bind(&CallbackCounter::Callback, base::Unretained(&counter)));
1177   scheduler()->ScheduleConfiguration(params);
1178
1179   scheduler()->OnConnectionStatusChange();
1180   scheduler()->OnConnectionStatusChange();
1181
1182   PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
1183 }
1184
1185 TEST_F(SyncSchedulerTest, PollFromCanaryAfterAuthError) {
1186   SyncShareTimes times;
1187   TimeDelta poll(TimeDelta::FromMilliseconds(15));
1188   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1189
1190   ::testing::InSequence seq;
1191   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1192       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1193            RecordSyncShareMultiple(&times, kMinNumSamples)));
1194
1195   connection()->SetServerStatus(HttpResponse::SYNC_AUTH_ERROR);
1196   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1197
1198   // Run to wait for polling.
1199   RunLoop();
1200
1201   // Normally OnCredentialsUpdated calls TryCanaryJob that doesn't run Poll,
1202   // but after poll finished with auth error from poll timer it should retry
1203   // poll once more
1204   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1205       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1206                       RecordSyncShare(&times)));
1207   scheduler()->OnCredentialsUpdated();
1208   connection()->SetServerStatus(HttpResponse::SERVER_CONNECTION_OK);
1209   StopSyncScheduler();
1210 }
1211
1212 }  // namespace syncer