sync/engine/sync_scheduler_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/bind.h"
   6 #include "base/callback.h"
   7 #include "base/compiler_specific.h"
   8 #include "base/memory/weak_ptr.h"
   9 #include "base/message_loop/message_loop.h"
  10 #include "base/test/test_timeouts.h"
  11 #include "sync/engine/backoff_delay_provider.h"
  12 #include "sync/engine/sync_scheduler_impl.h"
  13 #include "sync/engine/syncer.h"
  14 #include "sync/internal_api/public/base/cancelation_signal.h"
  15 #include "sync/internal_api/public/base/model_type_test_util.h"
  16 #include "sync/sessions/test_util.h"
  17 #include "sync/test/callback_counter.h"
  18 #include "sync/test/engine/fake_model_worker.h"
  19 #include "sync/test/engine/mock_connection_manager.h"
  20 #include "sync/test/engine/mock_nudge_handler.h"
  21 #include "sync/test/engine/test_directory_setter_upper.h"
  22 #include "sync/test/mock_invalidation.h"
  23 #include "sync/util/extensions_activity.h"
  24 #include "testing/gmock/include/gmock/gmock.h"
  25 #include "testing/gtest/include/gtest/gtest.h"
  26
  27 using base::TimeDelta;
  28 using base::TimeTicks;
  29 using testing::_;
  30 using testing::AtLeast;
  31 using testing::DoAll;
  32 using testing::Invoke;
  33 using testing::Mock;
  34 using testing::Return;
  35 using testing::WithArg;
  36 using testing::WithArgs;
  37 using testing::WithoutArgs;
  38
  39 namespace syncer {
  40 using sessions::SyncSession;
  41 using sessions::SyncSessionContext;
  42 using sync_pb::GetUpdatesCallerInfo;
  43
  44 class MockSyncer : public Syncer {
  45  public:
  46   MockSyncer();
  47   MOCK_METHOD3(NormalSyncShare, bool(ModelTypeSet,
  48                                      const sessions::NudgeTracker&,
  49                                      sessions::SyncSession*));
  50   MOCK_METHOD3(ConfigureSyncShare,
  51                bool(ModelTypeSet,
  52                     sync_pb::GetUpdatesCallerInfo::GetUpdatesSource,
  53                     SyncSession*));
  54   MOCK_METHOD2(PollSyncShare, bool(ModelTypeSet, sessions::SyncSession*));
  55 };
  56
  57 MockSyncer::MockSyncer()
  58   : Syncer(NULL) {}
  59
  60 typedef std::vector<TimeTicks> SyncShareTimes;
  61
  62 void QuitLoopNow() {
  63   // We use QuitNow() instead of Quit() as the latter may get stalled
  64   // indefinitely in the presence of repeated timers with low delays
  65   // and a slow test (e.g., ThrottlingDoesThrottle [which has a poll
  66   // delay of 5ms] run under TSAN on the trybots).
  67   base::MessageLoop::current()->QuitNow();
  68 }
  69
  70 void RunLoop() {
  71   base::MessageLoop::current()->Run();
  72 }
  73
  74 void PumpLoop() {
  75   // Do it this way instead of RunAllPending to pump loop exactly once
  76   // (necessary in the presence of timers; see comment in
  77   // QuitLoopNow).
  78   base::MessageLoop::current()->PostTask(FROM_HERE, base::Bind(&QuitLoopNow));
  79   RunLoop();
  80 }
  81
  82 void PumpLoopFor(base::TimeDelta time) {
  83   // Allow the loop to run for the specified amount of time.
  84   base::MessageLoop::current()->PostDelayedTask(
  85       FROM_HERE, base::Bind(&QuitLoopNow), time);
  86   RunLoop();
  87 }
  88
  89 ModelSafeRoutingInfo TypesToRoutingInfo(ModelTypeSet types) {
  90   ModelSafeRoutingInfo routes;
  91   for (ModelTypeSet::Iterator iter = types.First(); iter.Good(); iter.Inc()) {
  92     routes[iter.Get()] = GROUP_PASSIVE;
  93   }
  94   return routes;
  95 }
  96
  97 // Convenient to use in tests wishing to analyze SyncShare calls over time.
  98 static const size_t kMinNumSamples = 5;
  99 class SyncSchedulerTest : public testing::Test {
 100  public:
 101   SyncSchedulerTest() : syncer_(NULL), delay_(NULL), weak_ptr_factory_(this) {}
 102
 103   class MockDelayProvider : public BackoffDelayProvider {
 104    public:
 105     MockDelayProvider() : BackoffDelayProvider(
 106         TimeDelta::FromSeconds(kInitialBackoffRetrySeconds),
 107         TimeDelta::FromSeconds(kInitialBackoffImmediateRetrySeconds)) {
 108     }
 109
 110     MOCK_METHOD1(GetDelay, TimeDelta(const TimeDelta&));
 111   };
 112
 113   virtual void SetUp() {
 114     dir_maker_.SetUp();
 115     syncer_ = new testing::StrictMock<MockSyncer>();
 116     delay_ = NULL;
 117     extensions_activity_ = new ExtensionsActivity();
 118
 119     routing_info_[BOOKMARKS] = GROUP_UI;
 120     routing_info_[AUTOFILL] = GROUP_DB;
 121     routing_info_[THEMES] = GROUP_UI;
 122     routing_info_[NIGORI] = GROUP_PASSIVE;
 123
 124     workers_.clear();
 125     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_UI)));
 126     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_DB)));
 127     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_PASSIVE)));
 128
 129     connection_.reset(new MockConnectionManager(directory(),
 130                                                 &cancelation_signal_));
 131     connection_->SetServerReachable();
 132
 133     model_type_registry_.reset(
 134         new ModelTypeRegistry(workers_, directory(), &mock_nudge_handler_));
 135
 136     context_.reset(new SyncSessionContext(
 137             connection_.get(), directory(),
 138             extensions_activity_.get(),
 139             std::vector<SyncEngineEventListener*>(), NULL,
 140             model_type_registry_.get(),
 141             true,  // enable keystore encryption
 142             false,  // force enable pre-commit GU avoidance
 143             "fake_invalidator_client_id"));
 144     context_->SetRoutingInfo(routing_info_);
 145     context_->set_notifications_enabled(true);
 146     context_->set_account_name("Test");
 147     scheduler_.reset(
 148         new SyncSchedulerImpl("TestSyncScheduler",
 149             BackoffDelayProvider::FromDefaults(),
 150             context(),
 151             syncer_));
 152   }
 153
 154   SyncSchedulerImpl* scheduler() { return scheduler_.get(); }
 155   const ModelSafeRoutingInfo& routing_info() { return routing_info_; }
 156   MockSyncer* syncer() { return syncer_; }
 157   MockDelayProvider* delay() { return delay_; }
 158   MockConnectionManager* connection() { return connection_.get(); }
 159   TimeDelta zero() { return TimeDelta::FromSeconds(0); }
 160   TimeDelta timeout() {
 161     return TestTimeouts::action_timeout();
 162   }
 163
 164   virtual void TearDown() {
 165     PumpLoop();
 166     scheduler_.reset();
 167     PumpLoop();
 168     dir_maker_.TearDown();
 169   }
 170
 171   void AnalyzePollRun(const SyncShareTimes& times, size_t min_num_samples,
 172       const TimeTicks& optimal_start, const TimeDelta& poll_interval) {
 173     EXPECT_GE(times.size(), min_num_samples);
 174     for (size_t i = 0; i < times.size(); i++) {
 175       SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
 176       TimeTicks optimal_next_sync = optimal_start + poll_interval * i;
 177       EXPECT_GE(times[i], optimal_next_sync);
 178     }
 179   }
 180
 181   void DoQuitLoopNow() {
 182     QuitLoopNow();
 183   }
 184
 185   void StartSyncScheduler(SyncScheduler::Mode mode) {
 186     scheduler()->Start(mode);
 187   }
 188
 189   // This stops the scheduler synchronously.
 190   void StopSyncScheduler() {
 191     base::MessageLoop::current()->PostTask(
 192         FROM_HERE,
 193         base::Bind(&SyncSchedulerTest::DoQuitLoopNow,
 194                    weak_ptr_factory_.GetWeakPtr()));
 195     RunLoop();
 196   }
 197
 198   bool RunAndGetBackoff() {
 199     ModelTypeSet nudge_types(BOOKMARKS);
 200     StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 201
 202     scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
 203     RunLoop();
 204
 205     return scheduler()->IsBackingOff();
 206   }
 207
 208   void UseMockDelayProvider() {
 209     delay_ = new MockDelayProvider();
 210     scheduler_->delay_provider_.reset(delay_);
 211   }
 212
 213   SyncSessionContext* context() { return context_.get(); }
 214
 215   ModelTypeSet GetThrottledTypes() {
 216     return scheduler_->nudge_tracker_.GetThrottledTypes();
 217   }
 218
 219   base::TimeDelta GetRetryTimerDelay() {
 220     EXPECT_TRUE(scheduler_->retry_timer_.IsRunning());
 221     return scheduler_->retry_timer_.GetCurrentDelay();
 222   }
 223
 224   static scoped_ptr<InvalidationInterface> BuildInvalidation(
 225       int64 version,
 226       const std::string& payload) {
 227     return MockInvalidation::Build(version, payload)
 228         .PassAs<InvalidationInterface>();
 229   }
 230
 231  private:
 232   syncable::Directory* directory() {
 233     return dir_maker_.directory();
 234   }
 235
 236   base::MessageLoop loop_;
 237   TestDirectorySetterUpper dir_maker_;
 238   CancelationSignal cancelation_signal_;
 239   scoped_ptr<MockConnectionManager> connection_;
 240   scoped_ptr<ModelTypeRegistry> model_type_registry_;
 241   scoped_ptr<SyncSessionContext> context_;
 242   scoped_ptr<SyncSchedulerImpl> scheduler_;
 243   MockNudgeHandler mock_nudge_handler_;
 244   MockSyncer* syncer_;
 245   MockDelayProvider* delay_;
 246   std::vector<scoped_refptr<ModelSafeWorker> > workers_;
 247   scoped_refptr<ExtensionsActivity> extensions_activity_;
 248   ModelSafeRoutingInfo routing_info_;
 249   base::WeakPtrFactory<SyncSchedulerTest> weak_ptr_factory_;
 250 };
 251
 252 void RecordSyncShareImpl(SyncShareTimes* times) {
 253   times->push_back(TimeTicks::Now());
 254 }
 255
 256 ACTION_P(RecordSyncShare, times) {
 257   RecordSyncShareImpl(times);
 258   if (base::MessageLoop::current()->is_running())
 259     QuitLoopNow();
 260   return true;
 261 }
 262
 263 ACTION_P2(RecordSyncShareMultiple, times, quit_after) {
 264   RecordSyncShareImpl(times);
 265   EXPECT_LE(times->size(), quit_after);
 266   if (times->size() >= quit_after &&
 267       base::MessageLoop::current()->is_running()) {
 268     QuitLoopNow();
 269   }
 270   return true;
 271 }
 272
 273 ACTION_P(StopScheduler, scheduler) {
 274   scheduler->Stop();
 275 }
 276
 277 ACTION(AddFailureAndQuitLoopNow) {
 278   ADD_FAILURE();
 279   QuitLoopNow();
 280   return true;
 281 }
 282
 283 ACTION(QuitLoopNowAction) {
 284   QuitLoopNow();
 285   return true;
 286 }
 287
 288 // Test nudge scheduling.
 289 TEST_F(SyncSchedulerTest, Nudge) {
 290   SyncShareTimes times;
 291   ModelTypeSet model_types(BOOKMARKS);
 292
 293   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 294       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 295                       RecordSyncShare(&times)))
 296       .RetiresOnSaturation();
 297
 298   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 299
 300   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 301   RunLoop();
 302
 303   Mock::VerifyAndClearExpectations(syncer());
 304
 305   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 306   SyncShareTimes times2;
 307   model_types.Remove(BOOKMARKS);
 308   model_types.Put(AUTOFILL);
 309   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 310       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 311                       RecordSyncShare(&times2)));
 312   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 313   RunLoop();
 314 }
 315
 316 // Make sure a regular config command is scheduled fine in the absence of any
 317 // errors.
 318 TEST_F(SyncSchedulerTest, Config) {
 319   SyncShareTimes times;
 320   const ModelTypeSet model_types(BOOKMARKS);
 321
 322   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 323       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 324                       RecordSyncShare(&times)));
 325
 326   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 327
 328   CallbackCounter ready_counter;
 329   CallbackCounter retry_counter;
 330   ConfigurationParams params(
 331       GetUpdatesCallerInfo::RECONFIGURATION,
 332       model_types,
 333       TypesToRoutingInfo(model_types),
 334       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 335       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 336   scheduler()->ScheduleConfiguration(params);
 337   PumpLoop();
 338   ASSERT_EQ(1, ready_counter.times_called());
 339   ASSERT_EQ(0, retry_counter.times_called());
 340 }
 341
 342 // Simulate a failure and make sure the config request is retried.
 343 TEST_F(SyncSchedulerTest, ConfigWithBackingOff) {
 344   UseMockDelayProvider();
 345   EXPECT_CALL(*delay(), GetDelay(_))
 346       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
 347   SyncShareTimes times;
 348   const ModelTypeSet model_types(BOOKMARKS);
 349
 350   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 351
 352   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 353       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 354                       RecordSyncShare(&times)))
 355       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 356                       RecordSyncShare(&times)));
 357
 358   CallbackCounter ready_counter;
 359   CallbackCounter retry_counter;
 360   ConfigurationParams params(
 361       GetUpdatesCallerInfo::RECONFIGURATION,
 362       model_types,
 363       TypesToRoutingInfo(model_types),
 364       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 365       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 366   scheduler()->ScheduleConfiguration(params);
 367   RunLoop();
 368   ASSERT_EQ(0, ready_counter.times_called());
 369   ASSERT_EQ(1, retry_counter.times_called());
 370
 371   // RunLoop() will trigger TryCanaryJob which will retry configuration.
 372   // Since retry_task was already called it shouldn't be called again.
 373   RunLoop();
 374   ASSERT_EQ(0, ready_counter.times_called());
 375   ASSERT_EQ(1, retry_counter.times_called());
 376
 377   Mock::VerifyAndClearExpectations(syncer());
 378
 379   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 380       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 381                       RecordSyncShare(&times)));
 382   RunLoop();
 383
 384   ASSERT_EQ(1, ready_counter.times_called());
 385 }
 386
 387 // Simuilate SyncSchedulerImpl::Stop being called in the middle of Configure.
 388 // This can happen if server returns NOT_MY_BIRTHDAY.
 389 TEST_F(SyncSchedulerTest, ConfigWithStop) {
 390   UseMockDelayProvider();
 391   EXPECT_CALL(*delay(), GetDelay(_))
 392       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
 393   SyncShareTimes times;
 394   const ModelTypeSet model_types(BOOKMARKS);
 395
 396   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 397
 398   // Make ConfigureSyncShare call scheduler->Stop(). It is not supposed to call
 399   // retry_task or dereference configuration params.
 400   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 401       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 402                       StopScheduler(scheduler()),
 403                       RecordSyncShare(&times)));
 404
 405   CallbackCounter ready_counter;
 406   CallbackCounter retry_counter;
 407   ConfigurationParams params(
 408       GetUpdatesCallerInfo::RECONFIGURATION,
 409       model_types,
 410       TypesToRoutingInfo(model_types),
 411       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 412       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 413   scheduler()->ScheduleConfiguration(params);
 414   PumpLoop();
 415   ASSERT_EQ(0, ready_counter.times_called());
 416   ASSERT_EQ(0, retry_counter.times_called());
 417 }
 418
 419 // Issue a nudge when the config has failed. Make sure both the config and
 420 // nudge are executed.
 421 TEST_F(SyncSchedulerTest, NudgeWithConfigWithBackingOff) {
 422   const ModelTypeSet model_types(BOOKMARKS);
 423   UseMockDelayProvider();
 424   EXPECT_CALL(*delay(), GetDelay(_))
 425       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(50)));
 426   SyncShareTimes times;
 427
 428   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 429
 430   // Request a configure and make sure it fails.
 431   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 432       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 433                       RecordSyncShare(&times)));
 434   CallbackCounter ready_counter;
 435   CallbackCounter retry_counter;
 436   ConfigurationParams params(
 437       GetUpdatesCallerInfo::RECONFIGURATION,
 438       model_types,
 439       TypesToRoutingInfo(model_types),
 440       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 441       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 442   scheduler()->ScheduleConfiguration(params);
 443   RunLoop();
 444   ASSERT_EQ(0, ready_counter.times_called());
 445   ASSERT_EQ(1, retry_counter.times_called());
 446   Mock::VerifyAndClearExpectations(syncer());
 447
 448   // Ask for a nudge while dealing with repeated configure failure.
 449   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 450       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 451                       RecordSyncShare(&times)));
 452   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 453   RunLoop();
 454   // Note that we're not RunLoop()ing for the NUDGE we just scheduled, but
 455   // for the first retry attempt from the config job (after
 456   // waiting ~+/- 50ms).
 457   Mock::VerifyAndClearExpectations(syncer());
 458   ASSERT_EQ(0, ready_counter.times_called());
 459
 460   // Let the next configure retry succeed.
 461   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 462       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 463                       RecordSyncShare(&times)));
 464   RunLoop();
 465
 466   // Now change the mode so nudge can execute.
 467   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 468       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 469                       RecordSyncShare(&times)));
 470   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 471   PumpLoop();
 472 }
 473
 474 // Test that nudges are coalesced.
 475 TEST_F(SyncSchedulerTest, NudgeCoalescing) {
 476   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 477
 478   SyncShareTimes times;
 479   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 480       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 481                       RecordSyncShare(&times)));
 482   const ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3(THEMES);
 483   TimeDelta delay = zero();
 484   TimeTicks optimal_time = TimeTicks::Now() + delay;
 485   scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
 486   scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
 487   RunLoop();
 488
 489   ASSERT_EQ(1U, times.size());
 490   EXPECT_GE(times[0], optimal_time);
 491
 492   Mock::VerifyAndClearExpectations(syncer());
 493
 494   SyncShareTimes times2;
 495   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 496       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 497                       RecordSyncShare(&times2)));
 498   scheduler()->ScheduleLocalNudge(zero(), types3, FROM_HERE);
 499   RunLoop();
 500 }
 501
 502 // Test that nudges are coalesced.
 503 TEST_F(SyncSchedulerTest, NudgeCoalescingWithDifferentTimings) {
 504   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 505
 506   SyncShareTimes times;
 507   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 508       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 509                       RecordSyncShare(&times)));
 510   ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3;
 511
 512   // Create a huge time delay.
 513   TimeDelta delay = TimeDelta::FromDays(1);
 514
 515   scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
 516   scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
 517
 518   TimeTicks min_time = TimeTicks::Now();
 519   TimeTicks max_time = TimeTicks::Now() + delay;
 520
 521   RunLoop();
 522   Mock::VerifyAndClearExpectations(syncer());
 523
 524   // Make sure the sync happened at the right time.
 525   ASSERT_EQ(1U, times.size());
 526   EXPECT_GE(times[0], min_time);
 527   EXPECT_LE(times[0], max_time);
 528 }
 529
 530 // Test nudge scheduling.
 531 TEST_F(SyncSchedulerTest, NudgeWithStates) {
 532   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 533
 534   SyncShareTimes times1;
 535   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 536       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 537                       RecordSyncShare(&times1)))
 538       .RetiresOnSaturation();
 539   scheduler()->ScheduleInvalidationNudge(
 540       zero(), BOOKMARKS, BuildInvalidation(10, "test"), FROM_HERE);
 541   RunLoop();
 542
 543   Mock::VerifyAndClearExpectations(syncer());
 544
 545   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 546   SyncShareTimes times2;
 547   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 548       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 549                       RecordSyncShare(&times2)));
 550   scheduler()->ScheduleInvalidationNudge(
 551       zero(), AUTOFILL, BuildInvalidation(10, "test2"), FROM_HERE);
 552   RunLoop();
 553 }
 554
 555 // Test that polling works as expected.
 556 TEST_F(SyncSchedulerTest, Polling) {
 557   SyncShareTimes times;
 558   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 559   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 560       .WillRepeatedly(
 561           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 562                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 563
 564   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
 565
 566   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 567   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 568
 569   // Run again to wait for polling.
 570   RunLoop();
 571
 572   StopSyncScheduler();
 573   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 574 }
 575
 576 // Test that the short poll interval is used.
 577 TEST_F(SyncSchedulerTest, PollNotificationsDisabled) {
 578   SyncShareTimes times;
 579   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 580   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 581       .WillRepeatedly(
 582           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 583                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 584
 585   scheduler()->OnReceivedShortPollIntervalUpdate(poll_interval);
 586   scheduler()->SetNotificationsEnabled(false);
 587
 588   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 589   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 590
 591   // Run again to wait for polling.
 592   RunLoop();
 593
 594   StopSyncScheduler();
 595   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 596 }
 597
 598 // Test that polling intervals are updated when needed.
 599 TEST_F(SyncSchedulerTest, PollIntervalUpdate) {
 600   SyncShareTimes times;
 601   TimeDelta poll1(TimeDelta::FromMilliseconds(120));
 602   TimeDelta poll2(TimeDelta::FromMilliseconds(30));
 603   scheduler()->OnReceivedLongPollIntervalUpdate(poll1);
 604   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 605       .WillOnce(DoAll(
 606           WithArgs<0,1>(
 607               sessions::test_util::SimulatePollIntervalUpdate(poll2)),
 608           Return(true)))
 609       .WillRepeatedly(
 610           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 611                 WithArg<1>(
 612                     RecordSyncShareMultiple(&times, kMinNumSamples))));
 613
 614   TimeTicks optimal_start = TimeTicks::Now() + poll1 + poll2;
 615   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 616
 617   // Run again to wait for polling.
 618   RunLoop();
 619
 620   StopSyncScheduler();
 621   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll2);
 622 }
 623
 624 // Test that the sessions commit delay is updated when needed.
 625 TEST_F(SyncSchedulerTest, SessionsCommitDelay) {
 626   SyncShareTimes times;
 627   TimeDelta delay1(TimeDelta::FromMilliseconds(120));
 628   TimeDelta delay2(TimeDelta::FromMilliseconds(30));
 629   scheduler()->OnReceivedSessionsCommitDelay(delay1);
 630
 631   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 632       .WillOnce(
 633           DoAll(
 634               WithArgs<0,1,2>(
 635                   sessions::test_util::SimulateSessionsCommitDelayUpdate(
 636                       delay2)),
 637               Invoke(sessions::test_util::SimulateNormalSuccess),
 638               QuitLoopNowAction()));
 639
 640   EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
 641   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 642
 643   EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
 644   const ModelTypeSet model_types(BOOKMARKS);
 645   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 646   RunLoop();
 647
 648   EXPECT_EQ(delay2, scheduler()->GetSessionsCommitDelay());
 649   StopSyncScheduler();
 650 }
 651
 652 // Test that no syncing occurs when throttled.
 653 TEST_F(SyncSchedulerTest, ThrottlingDoesThrottle) {
 654   const ModelTypeSet types(BOOKMARKS);
 655   TimeDelta poll(TimeDelta::FromMilliseconds(5));
 656   TimeDelta throttle(TimeDelta::FromMinutes(10));
 657   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 658
 659   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 660       .WillOnce(DoAll(
 661           WithArg<2>(sessions::test_util::SimulateThrottled(throttle)),
 662           Return(true)))
 663       .WillRepeatedly(AddFailureAndQuitLoopNow());
 664
 665   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 666
 667   scheduler()->ScheduleLocalNudge(
 668       TimeDelta::FromMicroseconds(1), types, FROM_HERE);
 669   PumpLoop();
 670
 671   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 672
 673   CallbackCounter ready_counter;
 674   CallbackCounter retry_counter;
 675   ConfigurationParams params(
 676       GetUpdatesCallerInfo::RECONFIGURATION,
 677       types,
 678       TypesToRoutingInfo(types),
 679       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 680       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 681   scheduler()->ScheduleConfiguration(params);
 682   PumpLoop();
 683   ASSERT_EQ(0, ready_counter.times_called());
 684   ASSERT_EQ(1, retry_counter.times_called());
 685
 686 }
 687
 688 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromPoll) {
 689   SyncShareTimes times;
 690   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 691   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 692   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 693
 694   ::testing::InSequence seq;
 695   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 696       .WillOnce(DoAll(
 697           WithArg<1>(sessions::test_util::SimulateThrottled(throttle1)),
 698           Return(true)))
 699       .RetiresOnSaturation();
 700   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 701       .WillRepeatedly(
 702           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 703                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 704
 705   TimeTicks optimal_start = TimeTicks::Now() + poll + throttle1;
 706   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 707
 708   // Run again to wait for polling.
 709   RunLoop();
 710
 711   StopSyncScheduler();
 712   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll);
 713 }
 714
 715 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromNudge) {
 716   SyncShareTimes times;
 717   TimeDelta poll(TimeDelta::FromDays(1));
 718   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 719   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 720
 721   ::testing::InSequence seq;
 722   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 723       .WillOnce(DoAll(
 724           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 725           Return(true)))
 726       .RetiresOnSaturation();
 727   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 728       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 729                       QuitLoopNowAction()));
 730
 731   const ModelTypeSet types(BOOKMARKS);
 732   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 733   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 734
 735   PumpLoop(); // To get PerformDelayedNudge called.
 736   PumpLoop(); // To get TrySyncSessionJob called
 737   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 738   RunLoop();
 739   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 740
 741   StopSyncScheduler();
 742 }
 743
 744 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromConfigure) {
 745   SyncShareTimes times;
 746   TimeDelta poll(TimeDelta::FromDays(1));
 747   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 748   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 749
 750   ::testing::InSequence seq;
 751   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 752       .WillOnce(DoAll(
 753           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 754           Return(true)))
 755       .RetiresOnSaturation();
 756   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 757       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 758                       QuitLoopNowAction()));
 759
 760   const ModelTypeSet types(BOOKMARKS);
 761   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 762
 763   CallbackCounter ready_counter;
 764   CallbackCounter retry_counter;
 765   ConfigurationParams params(
 766       GetUpdatesCallerInfo::RECONFIGURATION,
 767       types,
 768       TypesToRoutingInfo(types),
 769       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 770       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 771   scheduler()->ScheduleConfiguration(params);
 772   PumpLoop();
 773   EXPECT_EQ(0, ready_counter.times_called());
 774   EXPECT_EQ(1, retry_counter.times_called());
 775   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 776
 777   RunLoop();
 778   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 779
 780   StopSyncScheduler();
 781 }
 782
 783 TEST_F(SyncSchedulerTest, TypeThrottlingBlocksNudge) {
 784   UseMockDelayProvider();
 785   EXPECT_CALL(*delay(), GetDelay(_))
 786       .WillRepeatedly(Return(zero()));
 787
 788   TimeDelta poll(TimeDelta::FromDays(1));
 789   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 790   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 791
 792   const ModelTypeSet types(BOOKMARKS);
 793
 794   ::testing::InSequence seq;
 795   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 796       .WillOnce(DoAll(
 797           WithArg<2>(
 798               sessions::test_util::SimulateTypesThrottled(types, throttle1)),
 799           Return(true)))
 800       .RetiresOnSaturation();
 801
 802   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 803   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 804   PumpLoop(); // To get PerformDelayedNudge called.
 805   PumpLoop(); // To get TrySyncSessionJob called
 806   EXPECT_TRUE(GetThrottledTypes().HasAll(types));
 807
 808   // This won't cause a sync cycle because the types are throttled.
 809   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 810   PumpLoop();
 811
 812   StopSyncScheduler();
 813 }
 814
 815 TEST_F(SyncSchedulerTest, TypeThrottlingDoesBlockOtherSources) {
 816   UseMockDelayProvider();
 817   EXPECT_CALL(*delay(), GetDelay(_))
 818       .WillRepeatedly(Return(zero()));
 819
 820   SyncShareTimes times;
 821   TimeDelta poll(TimeDelta::FromDays(1));
 822   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 823   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 824
 825   const ModelTypeSet throttled_types(BOOKMARKS);
 826   const ModelTypeSet unthrottled_types(PREFERENCES);
 827
 828   ::testing::InSequence seq;
 829   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 830       .WillOnce(DoAll(
 831           WithArg<2>(
 832               sessions::test_util::SimulateTypesThrottled(
 833                   throttled_types, throttle1)),
 834           Return(true)))
 835       .RetiresOnSaturation();
 836
 837   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 838   scheduler()->ScheduleLocalNudge(zero(), throttled_types, FROM_HERE);
 839   PumpLoop(); // To get PerformDelayedNudge called.
 840   PumpLoop(); // To get TrySyncSessionJob called
 841   EXPECT_TRUE(GetThrottledTypes().HasAll(throttled_types));
 842
 843   // Ignore invalidations for throttled types.
 844   scheduler()->ScheduleInvalidationNudge(
 845       zero(), BOOKMARKS, BuildInvalidation(10, "test"), FROM_HERE);
 846   PumpLoop();
 847
 848   // Ignore refresh requests for throttled types.
 849   scheduler()->ScheduleLocalRefreshRequest(zero(), throttled_types, FROM_HERE);
 850   PumpLoop();
 851
 852   Mock::VerifyAndClearExpectations(syncer());
 853
 854   // Local nudges for non-throttled types will trigger a sync.
 855   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 856       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 857                             RecordSyncShare(&times)));
 858   scheduler()->ScheduleLocalNudge(zero(), unthrottled_types, FROM_HERE);
 859   RunLoop();
 860   Mock::VerifyAndClearExpectations(syncer());
 861
 862   StopSyncScheduler();
 863 }
 864
 865 // Test nudges / polls don't run in config mode and config tasks do.
 866 TEST_F(SyncSchedulerTest, ConfigurationMode) {
 867   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 868   SyncShareTimes times;
 869   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 870
 871   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 872
 873   const ModelTypeSet nudge_types(AUTOFILL);
 874   scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
 875   scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
 876
 877   const ModelTypeSet config_types(BOOKMARKS);
 878
 879   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 880       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 881                       RecordSyncShare(&times)))
 882       .RetiresOnSaturation();
 883   CallbackCounter ready_counter;
 884   CallbackCounter retry_counter;
 885   ConfigurationParams params(
 886       GetUpdatesCallerInfo::RECONFIGURATION,
 887       config_types,
 888       TypesToRoutingInfo(config_types),
 889       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 890       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 891   scheduler()->ScheduleConfiguration(params);
 892   RunLoop();
 893   ASSERT_EQ(1, ready_counter.times_called());
 894   ASSERT_EQ(0, retry_counter.times_called());
 895
 896   Mock::VerifyAndClearExpectations(syncer());
 897
 898   // Switch to NORMAL_MODE to ensure NUDGES were properly saved and run.
 899   scheduler()->OnReceivedLongPollIntervalUpdate(TimeDelta::FromDays(1));
 900   SyncShareTimes times2;
 901   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 902       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 903                       RecordSyncShare(&times2)));
 904
 905   // TODO(tim): Figure out how to remove this dangerous need to reset
 906   // routing info between mode switches.
 907   context()->SetRoutingInfo(routing_info());
 908   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 909
 910   RunLoop();
 911   Mock::VerifyAndClearExpectations(syncer());
 912 }
 913
 914 class BackoffTriggersSyncSchedulerTest : public SyncSchedulerTest {
 915   virtual void SetUp() {
 916     SyncSchedulerTest::SetUp();
 917     UseMockDelayProvider();
 918     EXPECT_CALL(*delay(), GetDelay(_))
 919         .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
 920   }
 921
 922   virtual void TearDown() {
 923     StopSyncScheduler();
 924     SyncSchedulerTest::TearDown();
 925   }
 926 };
 927
 928 // Have the sycner fail during commit.  Expect that the scheduler enters
 929 // backoff.
 930 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnce) {
 931   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 932       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
 933                       QuitLoopNowAction()));
 934   EXPECT_TRUE(RunAndGetBackoff());
 935 }
 936
 937 // Have the syncer fail during download updates and succeed on the first
 938 // retry.  Expect that this clears the backoff state.
 939 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadOnceThenSucceed) {
 940   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 941       .WillOnce(DoAll(
 942           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 943           Return(true)))
 944       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 945                       QuitLoopNowAction()));
 946   EXPECT_FALSE(RunAndGetBackoff());
 947 }
 948
 949 // Have the syncer fail during commit and succeed on the first retry.  Expect
 950 // that this clears the backoff state.
 951 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnceThenSucceed) {
 952   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 953       .WillOnce(DoAll(
 954           Invoke(sessions::test_util::SimulateCommitFailed),
 955           Return(true)))
 956       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 957                       QuitLoopNowAction()));
 958   EXPECT_FALSE(RunAndGetBackoff());
 959 }
 960
 961 // Have the syncer fail to download updates and fail again on the retry.
 962 // Expect this will leave the scheduler in backoff.
 963 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadTwice) {
 964   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 965       .WillOnce(DoAll(
 966           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 967           Return(true)))
 968       .WillRepeatedly(DoAll(
 969               Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 970               QuitLoopNowAction()));
 971   EXPECT_TRUE(RunAndGetBackoff());
 972 }
 973
 974 // Have the syncer fail to get the encryption key yet succeed in downloading
 975 // updates. Expect this will leave the scheduler in backoff.
 976 TEST_F(BackoffTriggersSyncSchedulerTest, FailGetEncryptionKey) {
 977   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 978       .WillOnce(DoAll(
 979           Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
 980           Return(true)))
 981       .WillRepeatedly(DoAll(
 982               Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
 983               QuitLoopNowAction()));
 984   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 985
 986   ModelTypeSet types(BOOKMARKS);
 987   CallbackCounter ready_counter;
 988   CallbackCounter retry_counter;
 989   ConfigurationParams params(
 990       GetUpdatesCallerInfo::RECONFIGURATION,
 991       types,
 992       TypesToRoutingInfo(types),
 993       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 994       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 995   scheduler()->ScheduleConfiguration(params);
 996   RunLoop();
 997
 998   EXPECT_TRUE(scheduler()->IsBackingOff());
 999 }
1000
1001 // Test that no polls or extraneous nudges occur when in backoff.
1002 TEST_F(SyncSchedulerTest, BackoffDropsJobs) {
1003   SyncShareTimes times;
1004   TimeDelta poll(TimeDelta::FromMilliseconds(5));
1005   const ModelTypeSet types(BOOKMARKS);
1006   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1007   UseMockDelayProvider();
1008
1009   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1010       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1011                       RecordSyncShareMultiple(&times, 1U)));
1012   EXPECT_CALL(*delay(), GetDelay(_)).
1013       WillRepeatedly(Return(TimeDelta::FromDays(1)));
1014
1015   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1016
1017   // This nudge should fail and put us into backoff.  Thanks to our mock
1018   // GetDelay() setup above, this will be a long backoff.
1019   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
1020   RunLoop();
1021
1022   // From this point forward, no SyncShare functions should be invoked.
1023   Mock::VerifyAndClearExpectations(syncer());
1024
1025   // Wait a while (10x poll interval) so a few poll jobs will be attempted.
1026   PumpLoopFor(poll * 10);
1027
1028   // Try (and fail) to schedule a nudge.
1029   scheduler()->ScheduleLocalNudge(
1030       base::TimeDelta::FromMilliseconds(1),
1031       types,
1032       FROM_HERE);
1033
1034   Mock::VerifyAndClearExpectations(syncer());
1035   Mock::VerifyAndClearExpectations(delay());
1036
1037   EXPECT_CALL(*delay(), GetDelay(_)).Times(0);
1038
1039   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1040
1041   CallbackCounter ready_counter;
1042   CallbackCounter retry_counter;
1043   ConfigurationParams params(
1044       GetUpdatesCallerInfo::RECONFIGURATION,
1045       types,
1046       TypesToRoutingInfo(types),
1047       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1048       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1049   scheduler()->ScheduleConfiguration(params);
1050   PumpLoop();
1051   ASSERT_EQ(0, ready_counter.times_called());
1052   ASSERT_EQ(1, retry_counter.times_called());
1053
1054 }
1055
1056 // Test that backoff is shaping traffic properly with consecutive errors.
1057 TEST_F(SyncSchedulerTest, BackoffElevation) {
1058   SyncShareTimes times;
1059   UseMockDelayProvider();
1060
1061   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_)).Times(kMinNumSamples)
1062       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1063           RecordSyncShareMultiple(&times, kMinNumSamples)));
1064
1065   const TimeDelta first = TimeDelta::FromSeconds(kInitialBackoffRetrySeconds);
1066   const TimeDelta second = TimeDelta::FromMilliseconds(2);
1067   const TimeDelta third = TimeDelta::FromMilliseconds(3);
1068   const TimeDelta fourth = TimeDelta::FromMilliseconds(4);
1069   const TimeDelta fifth = TimeDelta::FromMilliseconds(5);
1070   const TimeDelta sixth = TimeDelta::FromDays(1);
1071
1072   EXPECT_CALL(*delay(), GetDelay(first)).WillOnce(Return(second))
1073           .RetiresOnSaturation();
1074   EXPECT_CALL(*delay(), GetDelay(second)).WillOnce(Return(third))
1075           .RetiresOnSaturation();
1076   EXPECT_CALL(*delay(), GetDelay(third)).WillOnce(Return(fourth))
1077           .RetiresOnSaturation();
1078   EXPECT_CALL(*delay(), GetDelay(fourth)).WillOnce(Return(fifth))
1079           .RetiresOnSaturation();
1080   EXPECT_CALL(*delay(), GetDelay(fifth)).WillOnce(Return(sixth));
1081
1082   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1083
1084   // Run again with a nudge.
1085   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1086   RunLoop();
1087
1088   ASSERT_EQ(kMinNumSamples, times.size());
1089   EXPECT_GE(times[1] - times[0], second);
1090   EXPECT_GE(times[2] - times[1], third);
1091   EXPECT_GE(times[3] - times[2], fourth);
1092   EXPECT_GE(times[4] - times[3], fifth);
1093 }
1094
1095 // Test that things go back to normal once a retry makes forward progress.
1096 TEST_F(SyncSchedulerTest, BackoffRelief) {
1097   SyncShareTimes times;
1098   const TimeDelta poll(TimeDelta::FromMilliseconds(10));
1099   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1100   UseMockDelayProvider();
1101
1102   const TimeDelta backoff = TimeDelta::FromMilliseconds(5);
1103   EXPECT_CALL(*delay(), GetDelay(_)).WillOnce(Return(backoff));
1104
1105   // Optimal start for the post-backoff poll party.
1106   TimeTicks optimal_start = TimeTicks::Now();
1107   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1108
1109   // Kick off the test with a failed nudge.
1110   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1111       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1112                       RecordSyncShare(&times)));
1113   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1114   RunLoop();
1115   Mock::VerifyAndClearExpectations(syncer());
1116   TimeTicks optimal_job_time = optimal_start;
1117   ASSERT_EQ(1U, times.size());
1118   EXPECT_GE(times[0], optimal_job_time);
1119
1120   // The retry succeeds.
1121   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1122       .WillOnce(DoAll(
1123               Invoke(sessions::test_util::SimulateNormalSuccess),
1124               RecordSyncShare(&times)));
1125   RunLoop();
1126   Mock::VerifyAndClearExpectations(syncer());
1127   optimal_job_time = optimal_job_time + backoff;
1128   ASSERT_EQ(2U, times.size());
1129   EXPECT_GE(times[1], optimal_job_time);
1130
1131   // Now let the Poll timer do its thing.
1132   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1133       .WillRepeatedly(DoAll(
1134               Invoke(sessions::test_util::SimulatePollSuccess),
1135               RecordSyncShareMultiple(&times, kMinNumSamples)));
1136   RunLoop();
1137   Mock::VerifyAndClearExpectations(syncer());
1138   ASSERT_EQ(kMinNumSamples, times.size());
1139   for (size_t i = 2; i < times.size(); i++) {
1140     optimal_job_time = optimal_job_time + poll;
1141     SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
1142     EXPECT_GE(times[i], optimal_job_time);
1143   }
1144
1145   StopSyncScheduler();
1146 }
1147
1148 // Test that poll failures are ignored.  They should have no effect on
1149 // subsequent poll attempts, nor should they trigger a backoff/retry.
1150 TEST_F(SyncSchedulerTest, TransientPollFailure) {
1151   SyncShareTimes times;
1152   const TimeDelta poll_interval(TimeDelta::FromMilliseconds(1));
1153   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
1154   UseMockDelayProvider(); // Will cause test failure if backoff is initiated.
1155
1156   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1157       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollFailed),
1158                       RecordSyncShare(&times)))
1159       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1160                       RecordSyncShare(&times)));
1161
1162   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1163
1164   // Run the unsucessful poll. The failed poll should not trigger backoff.
1165   RunLoop();
1166   EXPECT_FALSE(scheduler()->IsBackingOff());
1167
1168   // Run the successful poll.
1169   RunLoop();
1170   EXPECT_FALSE(scheduler()->IsBackingOff());
1171 }
1172
1173 // Test that starting the syncer thread without a valid connection doesn't
1174 // break things when a connection is detected.
1175 TEST_F(SyncSchedulerTest, StartWhenNotConnected) {
1176   connection()->SetServerNotReachable();
1177   connection()->UpdateConnectionStatus();
1178   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1179     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1180                     Return(true)))
1181     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1182                     Return(true)));
1183   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1184
1185   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1186   // Should save the nudge for until after the server is reachable.
1187   base::MessageLoop::current()->RunUntilIdle();
1188
1189   scheduler()->OnConnectionStatusChange();
1190   connection()->SetServerReachable();
1191   connection()->UpdateConnectionStatus();
1192   base::MessageLoop::current()->RunUntilIdle();
1193 }
1194
1195 TEST_F(SyncSchedulerTest, ServerConnectionChangeDuringBackoff) {
1196   UseMockDelayProvider();
1197   EXPECT_CALL(*delay(), GetDelay(_))
1198       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1199
1200   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1201   connection()->SetServerNotReachable();
1202   connection()->UpdateConnectionStatus();
1203
1204   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1205     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1206                     Return(true)))
1207     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1208                     Return(true)));
1209
1210   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1211   PumpLoop(); // To get PerformDelayedNudge called.
1212   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1213   ASSERT_TRUE(scheduler()->IsBackingOff());
1214
1215   // Before we run the scheduled canary, trigger a server connection change.
1216   scheduler()->OnConnectionStatusChange();
1217   connection()->SetServerReachable();
1218   connection()->UpdateConnectionStatus();
1219   base::MessageLoop::current()->RunUntilIdle();
1220 }
1221
1222 // This was supposed to test the scenario where we receive a nudge while a
1223 // connection change canary is scheduled, but has not run yet.  Since we've made
1224 // the connection change canary synchronous, this is no longer possible.
1225 TEST_F(SyncSchedulerTest, ConnectionChangeCanaryPreemptedByNudge) {
1226   UseMockDelayProvider();
1227   EXPECT_CALL(*delay(), GetDelay(_))
1228       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1229
1230   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1231   connection()->SetServerNotReachable();
1232   connection()->UpdateConnectionStatus();
1233
1234   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1235     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1236                     Return(true)))
1237     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1238                     Return(true)))
1239     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1240                     QuitLoopNowAction()));
1241
1242   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1243
1244   PumpLoop(); // To get PerformDelayedNudge called.
1245   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1246   ASSERT_TRUE(scheduler()->IsBackingOff());
1247
1248   // Before we run the scheduled canary, trigger a server connection change.
1249   scheduler()->OnConnectionStatusChange();
1250   PumpLoop();
1251   connection()->SetServerReachable();
1252   connection()->UpdateConnectionStatus();
1253   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1254   base::MessageLoop::current()->RunUntilIdle();
1255 }
1256
1257 // Tests that we don't crash trying to run two canaries at once if we receive
1258 // extra connection status change notifications.  See crbug.com/190085.
1259 TEST_F(SyncSchedulerTest, DoubleCanaryInConfigure) {
1260   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
1261       .WillRepeatedly(DoAll(
1262               Invoke(sessions::test_util::SimulateConfigureConnectionFailure),
1263               Return(true)));
1264   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1265   connection()->SetServerNotReachable();
1266   connection()->UpdateConnectionStatus();
1267
1268   ModelTypeSet model_types(BOOKMARKS);
1269   CallbackCounter ready_counter;
1270   CallbackCounter retry_counter;
1271   ConfigurationParams params(
1272       GetUpdatesCallerInfo::RECONFIGURATION,
1273       model_types,
1274       TypesToRoutingInfo(model_types),
1275       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1276       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1277   scheduler()->ScheduleConfiguration(params);
1278
1279   scheduler()->OnConnectionStatusChange();
1280   scheduler()->OnConnectionStatusChange();
1281
1282   PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
1283 }
1284
1285 TEST_F(SyncSchedulerTest, PollFromCanaryAfterAuthError) {
1286   SyncShareTimes times;
1287   TimeDelta poll(TimeDelta::FromMilliseconds(15));
1288   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1289
1290   ::testing::InSequence seq;
1291   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1292       .WillRepeatedly(
1293           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1294                 RecordSyncShareMultiple(&times, kMinNumSamples)));
1295
1296   connection()->SetServerStatus(HttpResponse::SYNC_AUTH_ERROR);
1297   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1298
1299   // Run to wait for polling.
1300   RunLoop();
1301
1302   // Normally OnCredentialsUpdated calls TryCanaryJob that doesn't run Poll,
1303   // but after poll finished with auth error from poll timer it should retry
1304   // poll once more
1305   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1306       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1307                       RecordSyncShare(&times)));
1308   scheduler()->OnCredentialsUpdated();
1309   connection()->SetServerStatus(HttpResponse::SERVER_CONNECTION_OK);
1310   RunLoop();
1311   StopSyncScheduler();
1312 }
1313
1314 TEST_F(SyncSchedulerTest, SuccessfulRetry) {
1315   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1316
1317   SyncShareTimes times;
1318   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(1);
1319   scheduler()->OnReceivedGuRetryDelay(delay);
1320   EXPECT_EQ(delay, GetRetryTimerDelay());
1321
1322   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1323       .WillOnce(
1324           DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1325                 RecordSyncShare(&times)));
1326
1327   // Run to wait for retrying.
1328   RunLoop();
1329
1330   StopSyncScheduler();
1331 }
1332
1333 TEST_F(SyncSchedulerTest, FailedRetry) {
1334   UseMockDelayProvider();
1335   EXPECT_CALL(*delay(), GetDelay(_))
1336       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
1337
1338   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1339
1340   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(1);
1341   scheduler()->OnReceivedGuRetryDelay(delay);
1342
1343   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1344       .WillOnce(
1345           DoAll(Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
1346                 QuitLoopNowAction()));
1347
1348   // Run to wait for retrying.
1349   RunLoop();
1350
1351   EXPECT_TRUE(scheduler()->IsBackingOff());
1352   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1353       .WillOnce(
1354           DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1355                 QuitLoopNowAction()));
1356
1357   // Run to wait for second retrying.
1358   RunLoop();
1359
1360   StopSyncScheduler();
1361 }
1362
1363 ACTION_P2(VerifyRetryTimerDelay, scheduler_test, expected_delay) {
1364   EXPECT_EQ(expected_delay, scheduler_test->GetRetryTimerDelay());
1365 }
1366
1367 TEST_F(SyncSchedulerTest, ReceiveNewRetryDelay) {
1368   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1369
1370   SyncShareTimes times;
1371   base::TimeDelta delay1 = base::TimeDelta::FromMilliseconds(100);
1372   base::TimeDelta delay2 = base::TimeDelta::FromMilliseconds(200);
1373
1374   scheduler()->ScheduleLocalRefreshRequest(zero(), ModelTypeSet(BOOKMARKS),
1375                                            FROM_HERE);
1376   scheduler()->OnReceivedGuRetryDelay(delay1);
1377   EXPECT_EQ(delay1, GetRetryTimerDelay());
1378
1379   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1380       .WillOnce(DoAll(
1381           WithoutArgs(VerifyRetryTimerDelay(this, delay1)),
1382           WithArg<2>(sessions::test_util::SimulateGuRetryDelayCommand(delay2)),
1383           RecordSyncShare(&times)));
1384
1385   // Run nudge GU.
1386   RunLoop();
1387   EXPECT_EQ(delay2, GetRetryTimerDelay());
1388
1389   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1390       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1391                       RecordSyncShare(&times)));
1392
1393   // Run to wait for retrying.
1394   RunLoop();
1395
1396   StopSyncScheduler();
1397 }
1398
1399 }  // namespace syncer