sync/engine/sync_scheduler_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/bind.h"
   6 #include "base/callback.h"
   7 #include "base/compiler_specific.h"
   8 #include "base/memory/weak_ptr.h"
   9 #include "base/run_loop.h"
  10 #include "base/test/test_timeouts.h"
  11 #include "sync/engine/backoff_delay_provider.h"
  12 #include "sync/engine/sync_scheduler_impl.h"
  13 #include "sync/engine/syncer.h"
  14 #include "sync/internal_api/public/base/cancelation_signal.h"
  15 #include "sync/internal_api/public/base/model_type_test_util.h"
  16 #include "sync/sessions/test_util.h"
  17 #include "sync/test/callback_counter.h"
  18 #include "sync/test/engine/fake_model_worker.h"
  19 #include "sync/test/engine/mock_connection_manager.h"
  20 #include "sync/test/engine/mock_nudge_handler.h"
  21 #include "sync/test/engine/test_directory_setter_upper.h"
  22 #include "sync/test/mock_invalidation.h"
  23 #include "sync/util/extensions_activity.h"
  24 #include "testing/gmock/include/gmock/gmock.h"
  25 #include "testing/gtest/include/gtest/gtest.h"
  26
  27 using base::TimeDelta;
  28 using base::TimeTicks;
  29 using testing::_;
  30 using testing::AtLeast;
  31 using testing::DoAll;
  32 using testing::Invoke;
  33 using testing::Mock;
  34 using testing::Return;
  35 using testing::WithArg;
  36 using testing::WithArgs;
  37 using testing::WithoutArgs;
  38
  39 namespace syncer {
  40 using sessions::SyncSession;
  41 using sessions::SyncSessionContext;
  42 using sync_pb::GetUpdatesCallerInfo;
  43
  44 class MockSyncer : public Syncer {
  45  public:
  46   MockSyncer();
  47   MOCK_METHOD3(NormalSyncShare,
  48                bool(ModelTypeSet,
  49                     sessions::NudgeTracker*,
  50                     sessions::SyncSession*));
  51   MOCK_METHOD3(ConfigureSyncShare,
  52                bool(ModelTypeSet,
  53                     sync_pb::GetUpdatesCallerInfo::GetUpdatesSource,
  54                     SyncSession*));
  55   MOCK_METHOD2(PollSyncShare, bool(ModelTypeSet, sessions::SyncSession*));
  56 };
  57
  58 MockSyncer::MockSyncer()
  59   : Syncer(NULL) {}
  60
  61 typedef std::vector<TimeTicks> SyncShareTimes;
  62
  63 void QuitLoopNow() {
  64   // We use QuitNow() instead of Quit() as the latter may get stalled
  65   // indefinitely in the presence of repeated timers with low delays
  66   // and a slow test (e.g., ThrottlingDoesThrottle [which has a poll
  67   // delay of 5ms] run under TSAN on the trybots).
  68   base::MessageLoop::current()->QuitNow();
  69 }
  70
  71 void RunLoop() {
  72   base::MessageLoop::current()->Run();
  73 }
  74
  75 void PumpLoop() {
  76   // Do it this way instead of RunAllPending to pump loop exactly once
  77   // (necessary in the presence of timers; see comment in
  78   // QuitLoopNow).
  79   base::MessageLoop::current()->PostTask(FROM_HERE, base::Bind(&QuitLoopNow));
  80   RunLoop();
  81 }
  82
  83 void PumpLoopFor(base::TimeDelta time) {
  84   // Allow the loop to run for the specified amount of time.
  85   base::MessageLoop::current()->PostDelayedTask(
  86       FROM_HERE, base::Bind(&QuitLoopNow), time);
  87   RunLoop();
  88 }
  89
  90 ModelSafeRoutingInfo TypesToRoutingInfo(ModelTypeSet types) {
  91   ModelSafeRoutingInfo routes;
  92   for (ModelTypeSet::Iterator iter = types.First(); iter.Good(); iter.Inc()) {
  93     routes[iter.Get()] = GROUP_PASSIVE;
  94   }
  95   return routes;
  96 }
  97
  98
  99 static const size_t kMinNumSamples = 5;
 100
 101 // Test harness for the SyncScheduler.  Test the delays and backoff timers used
 102 // in response to various events.
 103 //
 104 // These tests execute in real time with real timers.  We try to keep the
 105 // delays short, but there is a limit to how short we can make them.  The
 106 // timers on some platforms (ie. Windows) have a timer resolution greater than
 107 // 1ms.  Using 1ms delays may result in test flakiness.
 108 //
 109 // See crbug.com/402212 for more info.
 110 class SyncSchedulerTest : public testing::Test {
 111  public:
 112   SyncSchedulerTest() : syncer_(NULL), delay_(NULL), weak_ptr_factory_(this) {}
 113
 114   class MockDelayProvider : public BackoffDelayProvider {
 115    public:
 116     MockDelayProvider() : BackoffDelayProvider(
 117         TimeDelta::FromSeconds(kInitialBackoffRetrySeconds),
 118         TimeDelta::FromSeconds(kInitialBackoffImmediateRetrySeconds)) {
 119     }
 120
 121     MOCK_METHOD1(GetDelay, TimeDelta(const TimeDelta&));
 122   };
 123
 124   void SetUp() override {
 125     dir_maker_.SetUp();
 126     syncer_ = new testing::StrictMock<MockSyncer>();
 127     delay_ = NULL;
 128     extensions_activity_ = new ExtensionsActivity();
 129
 130     routing_info_[THEMES] = GROUP_UI;
 131     routing_info_[TYPED_URLS] = GROUP_DB;
 132     routing_info_[THEMES] = GROUP_UI;
 133     routing_info_[NIGORI] = GROUP_PASSIVE;
 134
 135     workers_.clear();
 136     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_UI)));
 137     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_DB)));
 138     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_PASSIVE)));
 139
 140     connection_.reset(new MockConnectionManager(directory(),
 141                                                 &cancelation_signal_));
 142     connection_->SetServerReachable();
 143
 144     model_type_registry_.reset(
 145         new ModelTypeRegistry(workers_, directory(), &mock_nudge_handler_));
 146
 147     context_.reset(new SyncSessionContext(
 148             connection_.get(), directory(),
 149             extensions_activity_.get(),
 150             std::vector<SyncEngineEventListener*>(), NULL,
 151             model_type_registry_.get(),
 152             true,  // enable keystore encryption
 153             false,  // force enable pre-commit GU avoidance
 154             "fake_invalidator_client_id"));
 155     context_->SetRoutingInfo(routing_info_);
 156     context_->set_notifications_enabled(true);
 157     context_->set_account_name("Test");
 158     scheduler_.reset(
 159         new SyncSchedulerImpl("TestSyncScheduler",
 160             BackoffDelayProvider::FromDefaults(),
 161             context(),
 162             syncer_));
 163     scheduler_->SetDefaultNudgeDelay(default_delay());
 164   }
 165
 166   SyncSchedulerImpl* scheduler() { return scheduler_.get(); }
 167   const ModelSafeRoutingInfo& routing_info() { return routing_info_; }
 168   MockSyncer* syncer() { return syncer_; }
 169   MockDelayProvider* delay() { return delay_; }
 170   MockConnectionManager* connection() { return connection_.get(); }
 171   TimeDelta default_delay() { return TimeDelta::FromSeconds(0); }
 172   TimeDelta timeout() {
 173     return TestTimeouts::action_timeout();
 174   }
 175
 176   void TearDown() override {
 177     PumpLoop();
 178     scheduler_.reset();
 179     PumpLoop();
 180     dir_maker_.TearDown();
 181   }
 182
 183   void AnalyzePollRun(const SyncShareTimes& times, size_t min_num_samples,
 184       const TimeTicks& optimal_start, const TimeDelta& poll_interval) {
 185     EXPECT_GE(times.size(), min_num_samples);
 186     for (size_t i = 0; i < times.size(); i++) {
 187       SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
 188       TimeTicks optimal_next_sync = optimal_start + poll_interval * i;
 189       EXPECT_GE(times[i], optimal_next_sync);
 190     }
 191   }
 192
 193   void DoQuitLoopNow() {
 194     QuitLoopNow();
 195   }
 196
 197   void StartSyncConfiguration() {
 198     scheduler()->Start(SyncScheduler::CONFIGURATION_MODE, base::Time());
 199   }
 200
 201   void StartSyncScheduler(base::Time last_poll_time) {
 202     scheduler()->Start(SyncScheduler::NORMAL_MODE, last_poll_time);
 203   }
 204
 205   // This stops the scheduler synchronously.
 206   void StopSyncScheduler() {
 207     base::MessageLoop::current()->PostTask(
 208         FROM_HERE,
 209         base::Bind(&SyncSchedulerTest::DoQuitLoopNow,
 210                    weak_ptr_factory_.GetWeakPtr()));
 211     RunLoop();
 212   }
 213
 214   bool RunAndGetBackoff() {
 215     ModelTypeSet nudge_types(THEMES);
 216     StartSyncScheduler(base::Time());
 217
 218     scheduler()->ScheduleLocalNudge(nudge_types, FROM_HERE);
 219     RunLoop();
 220
 221     return scheduler()->IsBackingOff();
 222   }
 223
 224   void UseMockDelayProvider() {
 225     delay_ = new MockDelayProvider();
 226     scheduler_->delay_provider_.reset(delay_);
 227   }
 228
 229   SyncSessionContext* context() { return context_.get(); }
 230
 231   ModelTypeSet GetThrottledTypes() {
 232     return scheduler_->nudge_tracker_.GetThrottledTypes();
 233   }
 234
 235   base::TimeDelta GetRetryTimerDelay() {
 236     EXPECT_TRUE(scheduler_->retry_timer_.IsRunning());
 237     return scheduler_->retry_timer_.GetCurrentDelay();
 238   }
 239
 240   static scoped_ptr<InvalidationInterface> BuildInvalidation(
 241       int64 version,
 242       const std::string& payload) {
 243     return MockInvalidation::Build(version, payload);
 244   }
 245
 246  private:
 247   syncable::Directory* directory() {
 248     return dir_maker_.directory();
 249   }
 250
 251   base::MessageLoop loop_;
 252   TestDirectorySetterUpper dir_maker_;
 253   CancelationSignal cancelation_signal_;
 254   scoped_ptr<MockConnectionManager> connection_;
 255   scoped_ptr<ModelTypeRegistry> model_type_registry_;
 256   scoped_ptr<SyncSessionContext> context_;
 257   scoped_ptr<SyncSchedulerImpl> scheduler_;
 258   MockNudgeHandler mock_nudge_handler_;
 259   MockSyncer* syncer_;
 260   MockDelayProvider* delay_;
 261   std::vector<scoped_refptr<ModelSafeWorker> > workers_;
 262   scoped_refptr<ExtensionsActivity> extensions_activity_;
 263   ModelSafeRoutingInfo routing_info_;
 264   base::WeakPtrFactory<SyncSchedulerTest> weak_ptr_factory_;
 265 };
 266
 267 void RecordSyncShareImpl(SyncShareTimes* times) {
 268   times->push_back(TimeTicks::Now());
 269 }
 270
 271 ACTION_P2(RecordSyncShare, times, success) {
 272   RecordSyncShareImpl(times);
 273   if (base::MessageLoop::current()->is_running())
 274     QuitLoopNow();
 275   return success;
 276 }
 277
 278 ACTION_P3(RecordSyncShareMultiple, times, quit_after, success) {
 279   RecordSyncShareImpl(times);
 280   EXPECT_LE(times->size(), quit_after);
 281   if (times->size() >= quit_after &&
 282       base::MessageLoop::current()->is_running()) {
 283     QuitLoopNow();
 284   }
 285   return success;
 286 }
 287
 288 ACTION_P(StopScheduler, scheduler) {
 289   scheduler->Stop();
 290 }
 291
 292 ACTION(AddFailureAndQuitLoopNow) {
 293   ADD_FAILURE();
 294   QuitLoopNow();
 295   return true;
 296 }
 297
 298 ACTION_P(QuitLoopNowAction, success) {
 299   QuitLoopNow();
 300   return success;
 301 }
 302
 303 // Test nudge scheduling.
 304 TEST_F(SyncSchedulerTest, Nudge) {
 305   SyncShareTimes times;
 306   ModelTypeSet model_types(THEMES);
 307
 308   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 309       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 310                       RecordSyncShare(&times, true)))
 311       .RetiresOnSaturation();
 312
 313   StartSyncScheduler(base::Time());
 314
 315   scheduler()->ScheduleLocalNudge(model_types, FROM_HERE);
 316   RunLoop();
 317
 318   Mock::VerifyAndClearExpectations(syncer());
 319
 320   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 321   SyncShareTimes times2;
 322   model_types.Remove(THEMES);
 323   model_types.Put(TYPED_URLS);
 324   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 325       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 326                       RecordSyncShare(&times2, true)));
 327   scheduler()->ScheduleLocalNudge(model_types, FROM_HERE);
 328   RunLoop();
 329 }
 330
 331 // Make sure a regular config command is scheduled fine in the absence of any
 332 // errors.
 333 TEST_F(SyncSchedulerTest, Config) {
 334   SyncShareTimes times;
 335   const ModelTypeSet model_types(THEMES);
 336
 337   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 338       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 339                       RecordSyncShare(&times, true)));
 340
 341   StartSyncConfiguration();
 342
 343   CallbackCounter ready_counter;
 344   CallbackCounter retry_counter;
 345   ConfigurationParams params(
 346       GetUpdatesCallerInfo::RECONFIGURATION,
 347       model_types,
 348       TypesToRoutingInfo(model_types),
 349       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 350       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 351   scheduler()->ScheduleConfiguration(params);
 352   PumpLoop();
 353   ASSERT_EQ(1, ready_counter.times_called());
 354   ASSERT_EQ(0, retry_counter.times_called());
 355 }
 356
 357 // Simulate a failure and make sure the config request is retried.
 358 TEST_F(SyncSchedulerTest, ConfigWithBackingOff) {
 359   UseMockDelayProvider();
 360   EXPECT_CALL(*delay(), GetDelay(_))
 361       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(20)));
 362   SyncShareTimes times;
 363   const ModelTypeSet model_types(THEMES);
 364
 365   StartSyncConfiguration();
 366
 367   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 368       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 369                       RecordSyncShare(&times, false)))
 370       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 371                       RecordSyncShare(&times, false)));
 372
 373   CallbackCounter ready_counter;
 374   CallbackCounter retry_counter;
 375   ConfigurationParams params(
 376       GetUpdatesCallerInfo::RECONFIGURATION,
 377       model_types,
 378       TypesToRoutingInfo(model_types),
 379       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 380       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 381   scheduler()->ScheduleConfiguration(params);
 382   RunLoop();
 383   ASSERT_EQ(0, ready_counter.times_called());
 384   ASSERT_EQ(1, retry_counter.times_called());
 385
 386   // RunLoop() will trigger TryCanaryJob which will retry configuration.
 387   // Since retry_task was already called it shouldn't be called again.
 388   RunLoop();
 389   ASSERT_EQ(0, ready_counter.times_called());
 390   ASSERT_EQ(1, retry_counter.times_called());
 391
 392   Mock::VerifyAndClearExpectations(syncer());
 393
 394   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 395       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 396                       RecordSyncShare(&times, true)));
 397   RunLoop();
 398
 399   ASSERT_EQ(1, ready_counter.times_called());
 400 }
 401
 402 // Simuilate SyncSchedulerImpl::Stop being called in the middle of Configure.
 403 // This can happen if server returns NOT_MY_BIRTHDAY.
 404 TEST_F(SyncSchedulerTest, ConfigWithStop) {
 405   UseMockDelayProvider();
 406   EXPECT_CALL(*delay(), GetDelay(_))
 407       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(20)));
 408   SyncShareTimes times;
 409   const ModelTypeSet model_types(THEMES);
 410
 411   StartSyncConfiguration();
 412
 413   // Make ConfigureSyncShare call scheduler->Stop(). It is not supposed to call
 414   // retry_task or dereference configuration params.
 415   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 416       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 417                       StopScheduler(scheduler()),
 418                       RecordSyncShare(&times, false)));
 419
 420   CallbackCounter ready_counter;
 421   CallbackCounter retry_counter;
 422   ConfigurationParams params(
 423       GetUpdatesCallerInfo::RECONFIGURATION,
 424       model_types,
 425       TypesToRoutingInfo(model_types),
 426       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 427       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 428   scheduler()->ScheduleConfiguration(params);
 429   PumpLoop();
 430   ASSERT_EQ(0, ready_counter.times_called());
 431   ASSERT_EQ(0, retry_counter.times_called());
 432 }
 433
 434 // Issue a nudge when the config has failed. Make sure both the config and
 435 // nudge are executed.
 436 TEST_F(SyncSchedulerTest, NudgeWithConfigWithBackingOff) {
 437   const ModelTypeSet model_types(THEMES);
 438   UseMockDelayProvider();
 439   EXPECT_CALL(*delay(), GetDelay(_))
 440       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(50)));
 441   SyncShareTimes times;
 442
 443   StartSyncConfiguration();
 444
 445   // Request a configure and make sure it fails.
 446   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 447       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 448                       RecordSyncShare(&times, false)));
 449   CallbackCounter ready_counter;
 450   CallbackCounter retry_counter;
 451   ConfigurationParams params(
 452       GetUpdatesCallerInfo::RECONFIGURATION,
 453       model_types,
 454       TypesToRoutingInfo(model_types),
 455       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 456       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 457   scheduler()->ScheduleConfiguration(params);
 458   RunLoop();
 459   ASSERT_EQ(0, ready_counter.times_called());
 460   ASSERT_EQ(1, retry_counter.times_called());
 461   Mock::VerifyAndClearExpectations(syncer());
 462
 463   // Ask for a nudge while dealing with repeated configure failure.
 464   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 465       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 466                       RecordSyncShare(&times, false)));
 467   scheduler()->ScheduleLocalNudge(model_types, FROM_HERE);
 468   RunLoop();
 469   // Note that we're not RunLoop()ing for the NUDGE we just scheduled, but
 470   // for the first retry attempt from the config job (after
 471   // waiting ~+/- 50ms).
 472   Mock::VerifyAndClearExpectations(syncer());
 473   ASSERT_EQ(0, ready_counter.times_called());
 474
 475   // Let the next configure retry succeed.
 476   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 477       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 478                       RecordSyncShare(&times, true)));
 479   RunLoop();
 480
 481   // Now change the mode so nudge can execute.
 482   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 483       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 484                       RecordSyncShare(&times, true)));
 485   StartSyncScheduler(base::Time());
 486   PumpLoop();
 487 }
 488
 489 // Test that nudges are coalesced.
 490 TEST_F(SyncSchedulerTest, NudgeCoalescing) {
 491   StartSyncScheduler(base::Time());
 492
 493   SyncShareTimes times;
 494   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 495       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 496                       RecordSyncShare(&times, true)));
 497   const ModelTypeSet types1(THEMES), types2(TYPED_URLS), types3(THEMES);
 498   TimeTicks optimal_time = TimeTicks::Now() + default_delay();
 499   scheduler()->ScheduleLocalNudge(types1, FROM_HERE);
 500   scheduler()->ScheduleLocalNudge(types2, FROM_HERE);
 501   RunLoop();
 502
 503   ASSERT_EQ(1U, times.size());
 504   EXPECT_GE(times[0], optimal_time);
 505
 506   Mock::VerifyAndClearExpectations(syncer());
 507
 508   SyncShareTimes times2;
 509   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 510       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 511                       RecordSyncShare(&times2, true)));
 512   scheduler()->ScheduleLocalNudge(types3, FROM_HERE);
 513   RunLoop();
 514 }
 515
 516 // Test that nudges are coalesced.
 517 TEST_F(SyncSchedulerTest, NudgeCoalescingWithDifferentTimings) {
 518   StartSyncScheduler(base::Time());
 519
 520   SyncShareTimes times;
 521   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 522       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 523                       RecordSyncShare(&times, true)));
 524   ModelTypeSet types1(THEMES), types2(TYPED_URLS), types3;
 525
 526   // Create a huge time delay.
 527   TimeDelta delay = TimeDelta::FromDays(1);
 528
 529   std::map<ModelType, TimeDelta> delay_map;
 530   delay_map[types1.First().Get()] = delay;
 531   scheduler()->OnReceivedCustomNudgeDelays(delay_map);
 532   scheduler()->ScheduleLocalNudge(types1, FROM_HERE);
 533   scheduler()->ScheduleLocalNudge(types2, FROM_HERE);
 534
 535   TimeTicks min_time = TimeTicks::Now();
 536   TimeTicks max_time = TimeTicks::Now() + delay;
 537
 538   RunLoop();
 539   Mock::VerifyAndClearExpectations(syncer());
 540
 541   // Make sure the sync happened at the right time.
 542   ASSERT_EQ(1U, times.size());
 543   EXPECT_GE(times[0], min_time);
 544   EXPECT_LE(times[0], max_time);
 545 }
 546
 547 // Test nudge scheduling.
 548 TEST_F(SyncSchedulerTest, NudgeWithStates) {
 549   StartSyncScheduler(base::Time());
 550
 551   SyncShareTimes times1;
 552   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 553       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 554                       RecordSyncShare(&times1, true)))
 555       .RetiresOnSaturation();
 556   scheduler()->ScheduleInvalidationNudge(
 557       THEMES, BuildInvalidation(10, "test"), FROM_HERE);
 558   RunLoop();
 559
 560   Mock::VerifyAndClearExpectations(syncer());
 561
 562   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 563   SyncShareTimes times2;
 564   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 565       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 566                       RecordSyncShare(&times2, true)));
 567   scheduler()->ScheduleInvalidationNudge(
 568       TYPED_URLS, BuildInvalidation(10, "test2"), FROM_HERE);
 569   RunLoop();
 570 }
 571
 572 // Test that polling works as expected.
 573 TEST_F(SyncSchedulerTest, Polling) {
 574   SyncShareTimes times;
 575   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 576   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 577       .WillRepeatedly(
 578           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 579                 RecordSyncShareMultiple(&times, kMinNumSamples, true)));
 580
 581   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
 582
 583   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 584   StartSyncScheduler(base::Time());
 585
 586   // Run again to wait for polling.
 587   RunLoop();
 588
 589   StopSyncScheduler();
 590   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 591 }
 592
 593 // Test that we reuse the previous poll time on startup, triggering the first
 594 // poll based on when the last one happened. Subsequent polls should have the
 595 // normal delay.
 596 TEST_F(SyncSchedulerTest, PollingPersistence) {
 597   SyncShareTimes times;
 598   // Use a large poll interval that wouldn't normally get hit on its own for
 599   // some time yet.
 600   TimeDelta poll_interval(TimeDelta::FromMilliseconds(500));
 601   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 602       .WillRepeatedly(
 603           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 604                 RecordSyncShareMultiple(&times, kMinNumSamples, true)));
 605
 606   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
 607
 608   // Set the start time to now, as the poll was overdue.
 609   TimeTicks optimal_start = TimeTicks::Now();
 610   StartSyncScheduler(base::Time::Now() - poll_interval);
 611
 612   // Run again to wait for polling.
 613   RunLoop();
 614
 615   StopSyncScheduler();
 616   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 617 }
 618
 619 // Test that if the persisted poll is in the future, it's ignored (the case
 620 // where the local time has been modified).
 621 TEST_F(SyncSchedulerTest, PollingPersistenceBadClock) {
 622   SyncShareTimes times;
 623   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 624   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 625       .WillRepeatedly(
 626           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 627                 RecordSyncShareMultiple(&times, kMinNumSamples, true)));
 628
 629   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
 630
 631   // Set the start time to |poll_interval| in the future.
 632   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 633   StartSyncScheduler(base::Time::Now() + base::TimeDelta::FromMinutes(10));
 634
 635   // Run again to wait for polling.
 636   RunLoop();
 637
 638   StopSyncScheduler();
 639   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 640 }
 641
 642 // Test that the short poll interval is used.
 643 TEST_F(SyncSchedulerTest, PollNotificationsDisabled) {
 644   SyncShareTimes times;
 645   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 646   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 647       .WillRepeatedly(
 648           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 649                 RecordSyncShareMultiple(&times, kMinNumSamples, true)));
 650
 651   scheduler()->OnReceivedShortPollIntervalUpdate(poll_interval);
 652   scheduler()->SetNotificationsEnabled(false);
 653
 654   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 655   StartSyncScheduler(base::Time());
 656
 657   // Run again to wait for polling.
 658   RunLoop();
 659
 660   StopSyncScheduler();
 661   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 662 }
 663
 664 // Test that polling intervals are updated when needed.
 665 TEST_F(SyncSchedulerTest, PollIntervalUpdate) {
 666   SyncShareTimes times;
 667   TimeDelta poll1(TimeDelta::FromMilliseconds(120));
 668   TimeDelta poll2(TimeDelta::FromMilliseconds(30));
 669   scheduler()->OnReceivedLongPollIntervalUpdate(poll1);
 670   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 671       .WillOnce(DoAll(
 672           WithArgs<0,1>(
 673               sessions::test_util::SimulatePollIntervalUpdate(poll2)),
 674           Return(true)))
 675       .WillRepeatedly(
 676           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 677                 WithArg<1>(
 678                     RecordSyncShareMultiple(&times, kMinNumSamples, true))));
 679
 680   TimeTicks optimal_start = TimeTicks::Now() + poll1 + poll2;
 681   StartSyncScheduler(base::Time());
 682
 683   // Run again to wait for polling.
 684   RunLoop();
 685
 686   StopSyncScheduler();
 687   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll2);
 688 }
 689
 690 // Test that no syncing occurs when throttled.
 691 TEST_F(SyncSchedulerTest, ThrottlingDoesThrottle) {
 692   const ModelTypeSet types(THEMES);
 693   TimeDelta poll(TimeDelta::FromMilliseconds(20));
 694   TimeDelta throttle(TimeDelta::FromMinutes(10));
 695   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 696
 697   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 698       .WillOnce(DoAll(
 699           WithArg<2>(sessions::test_util::SimulateThrottled(throttle)),
 700           Return(false)))
 701       .WillRepeatedly(AddFailureAndQuitLoopNow());
 702
 703   StartSyncScheduler(base::Time());
 704
 705   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
 706   PumpLoop();
 707
 708   StartSyncConfiguration();
 709
 710   CallbackCounter ready_counter;
 711   CallbackCounter retry_counter;
 712   ConfigurationParams params(
 713       GetUpdatesCallerInfo::RECONFIGURATION,
 714       types,
 715       TypesToRoutingInfo(types),
 716       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 717       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 718   scheduler()->ScheduleConfiguration(params);
 719   PumpLoop();
 720   ASSERT_EQ(0, ready_counter.times_called());
 721   ASSERT_EQ(1, retry_counter.times_called());
 722
 723 }
 724
 725 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromPoll) {
 726   SyncShareTimes times;
 727   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 728   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 729   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 730
 731   ::testing::InSequence seq;
 732   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 733       .WillOnce(DoAll(
 734           WithArg<1>(sessions::test_util::SimulateThrottled(throttle1)),
 735           Return(false)))
 736       .RetiresOnSaturation();
 737   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 738       .WillRepeatedly(
 739           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 740                 RecordSyncShareMultiple(&times, kMinNumSamples, true)));
 741
 742   TimeTicks optimal_start = TimeTicks::Now() + poll + throttle1;
 743   StartSyncScheduler(base::Time());
 744
 745   // Run again to wait for polling.
 746   RunLoop();
 747
 748   StopSyncScheduler();
 749   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll);
 750 }
 751
 752 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromNudge) {
 753   SyncShareTimes times;
 754   TimeDelta poll(TimeDelta::FromDays(1));
 755   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 756   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 757
 758   ::testing::InSequence seq;
 759   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 760       .WillOnce(DoAll(
 761           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 762           Return(false)))
 763       .RetiresOnSaturation();
 764   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 765       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 766                       QuitLoopNowAction(true)));
 767
 768   const ModelTypeSet types(THEMES);
 769   StartSyncScheduler(base::Time());
 770   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
 771
 772   PumpLoop(); // To get PerformDelayedNudge called.
 773   PumpLoop(); // To get TrySyncSessionJob called
 774   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 775   RunLoop();
 776   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 777
 778   StopSyncScheduler();
 779 }
 780
 781 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromConfigure) {
 782   SyncShareTimes times;
 783   TimeDelta poll(TimeDelta::FromDays(1));
 784   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 785   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 786
 787   ::testing::InSequence seq;
 788   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 789       .WillOnce(DoAll(
 790           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 791           Return(false)))
 792       .RetiresOnSaturation();
 793   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 794       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 795                       QuitLoopNowAction(true)));
 796
 797   const ModelTypeSet types(THEMES);
 798   StartSyncConfiguration();
 799
 800   CallbackCounter ready_counter;
 801   CallbackCounter retry_counter;
 802   ConfigurationParams params(
 803       GetUpdatesCallerInfo::RECONFIGURATION,
 804       types,
 805       TypesToRoutingInfo(types),
 806       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 807       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 808   scheduler()->ScheduleConfiguration(params);
 809   PumpLoop();
 810   EXPECT_EQ(0, ready_counter.times_called());
 811   EXPECT_EQ(1, retry_counter.times_called());
 812   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 813
 814   RunLoop();
 815   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 816
 817   StopSyncScheduler();
 818 }
 819
 820 TEST_F(SyncSchedulerTest, TypeThrottlingBlocksNudge) {
 821   UseMockDelayProvider();
 822   EXPECT_CALL(*delay(), GetDelay(_))
 823       .WillRepeatedly(Return(default_delay()));
 824
 825   TimeDelta poll(TimeDelta::FromDays(1));
 826   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 827   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 828
 829   const ModelTypeSet types(THEMES);
 830
 831   ::testing::InSequence seq;
 832   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 833       .WillOnce(DoAll(
 834           WithArg<2>(
 835               sessions::test_util::SimulateTypesThrottled(types, throttle1)),
 836           Return(false)))
 837       .RetiresOnSaturation();
 838
 839   StartSyncScheduler(base::Time());
 840   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
 841   PumpLoop(); // To get PerformDelayedNudge called.
 842   PumpLoop(); // To get TrySyncSessionJob called
 843   EXPECT_TRUE(GetThrottledTypes().HasAll(types));
 844
 845   // This won't cause a sync cycle because the types are throttled.
 846   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
 847   PumpLoop();
 848
 849   StopSyncScheduler();
 850 }
 851
 852 TEST_F(SyncSchedulerTest, TypeThrottlingDoesBlockOtherSources) {
 853   UseMockDelayProvider();
 854   EXPECT_CALL(*delay(), GetDelay(_))
 855       .WillRepeatedly(Return(default_delay()));
 856
 857   SyncShareTimes times;
 858   TimeDelta poll(TimeDelta::FromDays(1));
 859   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 860   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 861
 862   const ModelTypeSet throttled_types(THEMES);
 863   const ModelTypeSet unthrottled_types(PREFERENCES);
 864
 865   ::testing::InSequence seq;
 866   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 867       .WillOnce(DoAll(
 868           WithArg<2>(
 869               sessions::test_util::SimulateTypesThrottled(
 870                   throttled_types, throttle1)),
 871           Return(false)))
 872       .RetiresOnSaturation();
 873
 874   StartSyncScheduler(base::Time());
 875   scheduler()->ScheduleLocalNudge(throttled_types, FROM_HERE);
 876   PumpLoop(); // To get PerformDelayedNudge called.
 877   PumpLoop(); // To get TrySyncSessionJob called
 878   EXPECT_TRUE(GetThrottledTypes().HasAll(throttled_types));
 879
 880   // Ignore invalidations for throttled types.
 881   scheduler()->ScheduleInvalidationNudge(
 882       THEMES, BuildInvalidation(10, "test"), FROM_HERE);
 883   PumpLoop();
 884
 885   // Ignore refresh requests for throttled types.
 886   scheduler()->ScheduleLocalRefreshRequest(throttled_types, FROM_HERE);
 887   PumpLoop();
 888
 889   Mock::VerifyAndClearExpectations(syncer());
 890
 891   // Local nudges for non-throttled types will trigger a sync.
 892   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 893       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 894                             RecordSyncShare(&times, true)));
 895   scheduler()->ScheduleLocalNudge(unthrottled_types, FROM_HERE);
 896   RunLoop();
 897   Mock::VerifyAndClearExpectations(syncer());
 898
 899   StopSyncScheduler();
 900 }
 901
 902 // Test nudges / polls don't run in config mode and config tasks do.
 903 TEST_F(SyncSchedulerTest, ConfigurationMode) {
 904   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 905   SyncShareTimes times;
 906   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 907
 908   StartSyncConfiguration();
 909
 910   const ModelTypeSet nudge_types(TYPED_URLS);
 911   scheduler()->ScheduleLocalNudge(nudge_types, FROM_HERE);
 912   scheduler()->ScheduleLocalNudge(nudge_types, FROM_HERE);
 913
 914   const ModelTypeSet config_types(THEMES);
 915
 916   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 917       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 918                       RecordSyncShare(&times, true)))
 919       .RetiresOnSaturation();
 920   CallbackCounter ready_counter;
 921   CallbackCounter retry_counter;
 922   ConfigurationParams params(
 923       GetUpdatesCallerInfo::RECONFIGURATION,
 924       config_types,
 925       TypesToRoutingInfo(config_types),
 926       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 927       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 928   scheduler()->ScheduleConfiguration(params);
 929   RunLoop();
 930   ASSERT_EQ(1, ready_counter.times_called());
 931   ASSERT_EQ(0, retry_counter.times_called());
 932
 933   Mock::VerifyAndClearExpectations(syncer());
 934
 935   // Switch to NORMAL_MODE to ensure NUDGES were properly saved and run.
 936   scheduler()->OnReceivedLongPollIntervalUpdate(TimeDelta::FromDays(1));
 937   SyncShareTimes times2;
 938   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 939       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 940                       RecordSyncShare(&times2, true)));
 941
 942   // TODO(tim): Figure out how to remove this dangerous need to reset
 943   // routing info between mode switches.
 944   context()->SetRoutingInfo(routing_info());
 945   StartSyncScheduler(base::Time());
 946
 947   RunLoop();
 948   Mock::VerifyAndClearExpectations(syncer());
 949 }
 950
 951 class BackoffTriggersSyncSchedulerTest : public SyncSchedulerTest {
 952   void SetUp() override {
 953     SyncSchedulerTest::SetUp();
 954     UseMockDelayProvider();
 955     EXPECT_CALL(*delay(), GetDelay(_))
 956         .WillRepeatedly(Return(TimeDelta::FromMilliseconds(10)));
 957   }
 958
 959   void TearDown() override {
 960     StopSyncScheduler();
 961     SyncSchedulerTest::TearDown();
 962   }
 963 };
 964
 965 // Have the syncer fail during commit.  Expect that the scheduler enters
 966 // backoff.
 967 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnce) {
 968   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 969       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
 970                       QuitLoopNowAction(false)));
 971   EXPECT_TRUE(RunAndGetBackoff());
 972 }
 973
 974 // Have the syncer fail during download updates and succeed on the first
 975 // retry.  Expect that this clears the backoff state.
 976 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadOnceThenSucceed) {
 977   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 978       .WillOnce(DoAll(
 979           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 980           Return(false)))
 981       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 982                       QuitLoopNowAction(true)));
 983   EXPECT_FALSE(RunAndGetBackoff());
 984 }
 985
 986 // Have the syncer fail during commit and succeed on the first retry.  Expect
 987 // that this clears the backoff state.
 988 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnceThenSucceed) {
 989   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 990       .WillOnce(DoAll(
 991           Invoke(sessions::test_util::SimulateCommitFailed),
 992           Return(false)))
 993       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 994                       QuitLoopNowAction(true)));
 995   EXPECT_FALSE(RunAndGetBackoff());
 996 }
 997
 998 // Have the syncer fail to download updates and fail again on the retry.
 999 // Expect this will leave the scheduler in backoff.
1000 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadTwice) {
1001   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1002       .WillOnce(DoAll(
1003           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
1004           Return(false)))
1005       .WillRepeatedly(DoAll(
1006           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
1007           QuitLoopNowAction(false)));
1008   EXPECT_TRUE(RunAndGetBackoff());
1009 }
1010
1011 // Have the syncer fail to get the encryption key yet succeed in downloading
1012 // updates. Expect this will leave the scheduler in backoff.
1013 TEST_F(BackoffTriggersSyncSchedulerTest, FailGetEncryptionKey) {
1014   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
1015       .WillOnce(DoAll(
1016           Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
1017           Return(false)))
1018       .WillRepeatedly(DoAll(
1019           Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
1020           QuitLoopNowAction(false)));
1021   StartSyncConfiguration();
1022
1023   ModelTypeSet types(THEMES);
1024   CallbackCounter ready_counter;
1025   CallbackCounter retry_counter;
1026   ConfigurationParams params(
1027       GetUpdatesCallerInfo::RECONFIGURATION,
1028       types,
1029       TypesToRoutingInfo(types),
1030       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1031       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1032   scheduler()->ScheduleConfiguration(params);
1033   RunLoop();
1034
1035   EXPECT_TRUE(scheduler()->IsBackingOff());
1036 }
1037
1038 // Test that no polls or extraneous nudges occur when in backoff.
1039 TEST_F(SyncSchedulerTest, BackoffDropsJobs) {
1040   SyncShareTimes times;
1041   TimeDelta poll(TimeDelta::FromMilliseconds(10));
1042   const ModelTypeSet types(THEMES);
1043   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1044   UseMockDelayProvider();
1045
1046   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1047       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1048                       RecordSyncShareMultiple(&times, 1U, false)));
1049   EXPECT_CALL(*delay(), GetDelay(_)).
1050       WillRepeatedly(Return(TimeDelta::FromDays(1)));
1051
1052   StartSyncScheduler(base::Time());
1053
1054   // This nudge should fail and put us into backoff.  Thanks to our mock
1055   // GetDelay() setup above, this will be a long backoff.
1056   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
1057   RunLoop();
1058
1059   // From this point forward, no SyncShare functions should be invoked.
1060   Mock::VerifyAndClearExpectations(syncer());
1061
1062   // Wait a while (10x poll interval) so a few poll jobs will be attempted.
1063   PumpLoopFor(poll * 10);
1064
1065   // Try (and fail) to schedule a nudge.
1066   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
1067
1068   Mock::VerifyAndClearExpectations(syncer());
1069   Mock::VerifyAndClearExpectations(delay());
1070
1071   EXPECT_CALL(*delay(), GetDelay(_)).Times(0);
1072
1073   StartSyncConfiguration();
1074
1075   CallbackCounter ready_counter;
1076   CallbackCounter retry_counter;
1077   ConfigurationParams params(
1078       GetUpdatesCallerInfo::RECONFIGURATION,
1079       types,
1080       TypesToRoutingInfo(types),
1081       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1082       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1083   scheduler()->ScheduleConfiguration(params);
1084   PumpLoop();
1085   ASSERT_EQ(0, ready_counter.times_called());
1086   ASSERT_EQ(1, retry_counter.times_called());
1087
1088 }
1089
1090 // Test that backoff is shaping traffic properly with consecutive errors.
1091 TEST_F(SyncSchedulerTest, BackoffElevation) {
1092   SyncShareTimes times;
1093   UseMockDelayProvider();
1094
1095   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_)).Times(kMinNumSamples)
1096       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1097           RecordSyncShareMultiple(&times, kMinNumSamples, false)));
1098
1099   const TimeDelta first = TimeDelta::FromSeconds(kInitialBackoffRetrySeconds);
1100   const TimeDelta second = TimeDelta::FromMilliseconds(20);
1101   const TimeDelta third = TimeDelta::FromMilliseconds(30);
1102   const TimeDelta fourth = TimeDelta::FromMilliseconds(40);
1103   const TimeDelta fifth = TimeDelta::FromMilliseconds(50);
1104   const TimeDelta sixth = TimeDelta::FromDays(1);
1105
1106   EXPECT_CALL(*delay(), GetDelay(first)).WillOnce(Return(second))
1107           .RetiresOnSaturation();
1108   EXPECT_CALL(*delay(), GetDelay(second)).WillOnce(Return(third))
1109           .RetiresOnSaturation();
1110   EXPECT_CALL(*delay(), GetDelay(third)).WillOnce(Return(fourth))
1111           .RetiresOnSaturation();
1112   EXPECT_CALL(*delay(), GetDelay(fourth)).WillOnce(Return(fifth))
1113           .RetiresOnSaturation();
1114   EXPECT_CALL(*delay(), GetDelay(fifth)).WillOnce(Return(sixth));
1115
1116   StartSyncScheduler(base::Time());
1117
1118   // Run again with a nudge.
1119   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1120   RunLoop();
1121
1122   ASSERT_EQ(kMinNumSamples, times.size());
1123   EXPECT_GE(times[1] - times[0], second);
1124   EXPECT_GE(times[2] - times[1], third);
1125   EXPECT_GE(times[3] - times[2], fourth);
1126   EXPECT_GE(times[4] - times[3], fifth);
1127 }
1128
1129 // Test that things go back to normal once a retry makes forward progress.
1130 TEST_F(SyncSchedulerTest, BackoffRelief) {
1131   SyncShareTimes times;
1132   UseMockDelayProvider();
1133
1134   const TimeDelta backoff = TimeDelta::FromMilliseconds(10);
1135   EXPECT_CALL(*delay(), GetDelay(_)).WillOnce(Return(backoff));
1136
1137   // Optimal start for the post-backoff poll party.
1138   TimeTicks optimal_start = TimeTicks::Now();
1139   StartSyncScheduler(base::Time());
1140
1141   // Kick off the test with a failed nudge.
1142   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1143       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1144                       RecordSyncShare(&times, false)));
1145   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1146   RunLoop();
1147   Mock::VerifyAndClearExpectations(syncer());
1148   TimeTicks optimal_job_time = optimal_start;
1149   ASSERT_EQ(1U, times.size());
1150   EXPECT_GE(times[0], optimal_job_time);
1151
1152   // The retry succeeds.
1153   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1154       .WillOnce(DoAll(
1155               Invoke(sessions::test_util::SimulateNormalSuccess),
1156               RecordSyncShare(&times, true)));
1157   RunLoop();
1158   Mock::VerifyAndClearExpectations(syncer());
1159   optimal_job_time = optimal_job_time + backoff;
1160   ASSERT_EQ(2U, times.size());
1161   EXPECT_GE(times[1], optimal_job_time);
1162
1163   // Now let the Poll timer do its thing.
1164   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1165       .WillRepeatedly(DoAll(
1166               Invoke(sessions::test_util::SimulatePollSuccess),
1167               RecordSyncShareMultiple(&times, kMinNumSamples, true)));
1168   const TimeDelta poll(TimeDelta::FromMilliseconds(10));
1169   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1170
1171   // The new optimal time is now, since the desired poll should have happened
1172   // in the past.
1173   optimal_job_time = base::TimeTicks::Now();
1174   RunLoop();
1175   Mock::VerifyAndClearExpectations(syncer());
1176   ASSERT_EQ(kMinNumSamples, times.size());
1177   for (size_t i = 2; i < times.size(); i++) {
1178     SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
1179     EXPECT_GE(times[i], optimal_job_time);
1180     optimal_job_time = optimal_job_time + poll;
1181   }
1182
1183   StopSyncScheduler();
1184 }
1185
1186 // Test that poll failures are treated like any other failure. They should
1187 // result in retry with backoff.
1188 TEST_F(SyncSchedulerTest, TransientPollFailure) {
1189   SyncShareTimes times;
1190   const TimeDelta poll_interval(TimeDelta::FromMilliseconds(10));
1191   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
1192   UseMockDelayProvider(); // Will cause test failure if backoff is initiated.
1193   EXPECT_CALL(*delay(), GetDelay(_))
1194       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1195
1196   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1197       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollFailed),
1198                       RecordSyncShare(&times, false)))
1199       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1200                       RecordSyncShare(&times, true)));
1201
1202   StartSyncScheduler(base::Time());
1203
1204   // Run the unsucessful poll. The failed poll should not trigger backoff.
1205   RunLoop();
1206   EXPECT_TRUE(scheduler()->IsBackingOff());
1207
1208   // Run the successful poll.
1209   RunLoop();
1210   EXPECT_FALSE(scheduler()->IsBackingOff());
1211 }
1212
1213 // Test that starting the syncer thread without a valid connection doesn't
1214 // break things when a connection is detected.
1215 TEST_F(SyncSchedulerTest, StartWhenNotConnected) {
1216   connection()->SetServerNotReachable();
1217   connection()->UpdateConnectionStatus();
1218   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1219     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1220                     Return(false)))
1221     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1222                     Return(true)));
1223   StartSyncScheduler(base::Time());
1224
1225   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1226   // Should save the nudge for until after the server is reachable.
1227   base::RunLoop().RunUntilIdle();
1228
1229   scheduler()->OnConnectionStatusChange();
1230   connection()->SetServerReachable();
1231   connection()->UpdateConnectionStatus();
1232   base::RunLoop().RunUntilIdle();
1233 }
1234
1235 TEST_F(SyncSchedulerTest, ServerConnectionChangeDuringBackoff) {
1236   UseMockDelayProvider();
1237   EXPECT_CALL(*delay(), GetDelay(_))
1238       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1239
1240   StartSyncScheduler(base::Time());
1241   connection()->SetServerNotReachable();
1242   connection()->UpdateConnectionStatus();
1243
1244   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1245     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1246                     Return(false)))
1247     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1248                     Return(true)));
1249
1250   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1251   PumpLoop(); // To get PerformDelayedNudge called.
1252   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1253   ASSERT_TRUE(scheduler()->IsBackingOff());
1254
1255   // Before we run the scheduled canary, trigger a server connection change.
1256   scheduler()->OnConnectionStatusChange();
1257   connection()->SetServerReachable();
1258   connection()->UpdateConnectionStatus();
1259   base::RunLoop().RunUntilIdle();
1260 }
1261
1262 // This was supposed to test the scenario where we receive a nudge while a
1263 // connection change canary is scheduled, but has not run yet.  Since we've made
1264 // the connection change canary synchronous, this is no longer possible.
1265 TEST_F(SyncSchedulerTest, ConnectionChangeCanaryPreemptedByNudge) {
1266   UseMockDelayProvider();
1267   EXPECT_CALL(*delay(), GetDelay(_))
1268       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1269
1270   StartSyncScheduler(base::Time());
1271   connection()->SetServerNotReachable();
1272   connection()->UpdateConnectionStatus();
1273
1274   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1275     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1276                     Return(false)))
1277     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1278                     Return(true)))
1279     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1280                     QuitLoopNowAction(true)));
1281
1282   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1283
1284   PumpLoop(); // To get PerformDelayedNudge called.
1285   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1286   ASSERT_TRUE(scheduler()->IsBackingOff());
1287
1288   // Before we run the scheduled canary, trigger a server connection change.
1289   scheduler()->OnConnectionStatusChange();
1290   PumpLoop();
1291   connection()->SetServerReachable();
1292   connection()->UpdateConnectionStatus();
1293   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1294   base::RunLoop().RunUntilIdle();
1295 }
1296
1297 // Tests that we don't crash trying to run two canaries at once if we receive
1298 // extra connection status change notifications.  See crbug.com/190085.
1299 TEST_F(SyncSchedulerTest, DoubleCanaryInConfigure) {
1300   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
1301       .WillRepeatedly(DoAll(
1302               Invoke(sessions::test_util::SimulateConfigureConnectionFailure),
1303               Return(true)));
1304   StartSyncConfiguration();
1305   connection()->SetServerNotReachable();
1306   connection()->UpdateConnectionStatus();
1307
1308   ModelTypeSet model_types(THEMES);
1309   CallbackCounter ready_counter;
1310   CallbackCounter retry_counter;
1311   ConfigurationParams params(
1312       GetUpdatesCallerInfo::RECONFIGURATION,
1313       model_types,
1314       TypesToRoutingInfo(model_types),
1315       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1316       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1317   scheduler()->ScheduleConfiguration(params);
1318
1319   scheduler()->OnConnectionStatusChange();
1320   scheduler()->OnConnectionStatusChange();
1321
1322   PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
1323 }
1324
1325 TEST_F(SyncSchedulerTest, PollFromCanaryAfterAuthError) {
1326   SyncShareTimes times;
1327   TimeDelta poll(TimeDelta::FromMilliseconds(15));
1328   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1329
1330   ::testing::InSequence seq;
1331   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1332       .WillRepeatedly(
1333           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1334                 RecordSyncShareMultiple(&times, kMinNumSamples, true)));
1335
1336   connection()->SetServerStatus(HttpResponse::SYNC_AUTH_ERROR);
1337   StartSyncScheduler(base::Time());
1338
1339   // Run to wait for polling.
1340   RunLoop();
1341
1342   // Normally OnCredentialsUpdated calls TryCanaryJob that doesn't run Poll,
1343   // but after poll finished with auth error from poll timer it should retry
1344   // poll once more
1345   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1346       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1347                       RecordSyncShare(&times, true)));
1348   scheduler()->OnCredentialsUpdated();
1349   connection()->SetServerStatus(HttpResponse::SERVER_CONNECTION_OK);
1350   RunLoop();
1351   StopSyncScheduler();
1352 }
1353
1354 TEST_F(SyncSchedulerTest, SuccessfulRetry) {
1355   StartSyncScheduler(base::Time());
1356
1357   SyncShareTimes times;
1358   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(10);
1359   scheduler()->OnReceivedGuRetryDelay(delay);
1360   EXPECT_EQ(delay, GetRetryTimerDelay());
1361
1362   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1363       .WillOnce(
1364           DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1365                 RecordSyncShare(&times, true)));
1366
1367   // Run to wait for retrying.
1368   RunLoop();
1369
1370   StopSyncScheduler();
1371 }
1372
1373 TEST_F(SyncSchedulerTest, FailedRetry) {
1374   SyncShareTimes times;
1375
1376   UseMockDelayProvider();
1377   EXPECT_CALL(*delay(), GetDelay(_))
1378       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(10)));
1379
1380   StartSyncScheduler(base::Time());
1381
1382   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(10);
1383   scheduler()->OnReceivedGuRetryDelay(delay);
1384
1385   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1386       .WillOnce(
1387           DoAll(Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
1388                 RecordSyncShare(&times, false)));
1389
1390   // Run to wait for retrying.
1391   RunLoop();
1392
1393   EXPECT_TRUE(scheduler()->IsBackingOff());
1394   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1395       .WillOnce(
1396           DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1397                 RecordSyncShare(&times, true)));
1398
1399   // Run to wait for second retrying.
1400   RunLoop();
1401
1402   StopSyncScheduler();
1403 }
1404
1405 ACTION_P2(VerifyRetryTimerDelay, scheduler_test, expected_delay) {
1406   EXPECT_EQ(expected_delay, scheduler_test->GetRetryTimerDelay());
1407 }
1408
1409 TEST_F(SyncSchedulerTest, ReceiveNewRetryDelay) {
1410   StartSyncScheduler(base::Time());
1411
1412   SyncShareTimes times;
1413   base::TimeDelta delay1 = base::TimeDelta::FromMilliseconds(100);
1414   base::TimeDelta delay2 = base::TimeDelta::FromMilliseconds(200);
1415
1416   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1417   scheduler()->OnReceivedGuRetryDelay(delay1);
1418   EXPECT_EQ(delay1, GetRetryTimerDelay());
1419
1420   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1421       .WillOnce(DoAll(
1422           WithoutArgs(VerifyRetryTimerDelay(this, delay1)),
1423           WithArg<2>(sessions::test_util::SimulateGuRetryDelayCommand(delay2)),
1424           RecordSyncShare(&times, true)));
1425
1426   // Run nudge GU.
1427   RunLoop();
1428   EXPECT_EQ(delay2, GetRetryTimerDelay());
1429
1430   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1431       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1432                       RecordSyncShare(&times, true)));
1433
1434   // Run to wait for retrying.
1435   RunLoop();
1436
1437   StopSyncScheduler();
1438 }
1439
1440 TEST_F(SyncSchedulerTest, ScheduleClearServerData_Succeeds) {
1441   StartSyncConfiguration();
1442   scheduler()->Start(SyncScheduler::CLEAR_SERVER_DATA_MODE, base::Time());
1443   CallbackCounter success_counter;
1444   ClearParams params(base::Bind(&CallbackCounter::Callback,
1445                                 base::Unretained(&success_counter)));
1446   scheduler()->ScheduleClearServerData(params);
1447   PumpLoop();
1448   ASSERT_EQ(1, success_counter.times_called());
1449 }
1450
1451 TEST_F(SyncSchedulerTest, ScheduleClearServerData_FailsRetriesSucceeds) {
1452   UseMockDelayProvider();
1453   TimeDelta delta(TimeDelta::FromMilliseconds(20));
1454   EXPECT_CALL(*delay(), GetDelay(_)).WillRepeatedly(Return(delta));
1455
1456   StartSyncConfiguration();
1457   scheduler()->Start(SyncScheduler::CLEAR_SERVER_DATA_MODE, base::Time());
1458   CallbackCounter success_counter;
1459   ClearParams params(base::Bind(&CallbackCounter::Callback,
1460                                 base::Unretained(&success_counter)));
1461
1462   // Next request will fail.
1463   connection()->SetServerNotReachable();
1464   scheduler()->ScheduleClearServerData(params);
1465   PumpLoop();
1466   ASSERT_EQ(0, success_counter.times_called());
1467   ASSERT_TRUE(scheduler()->IsBackingOff());
1468
1469   // Now succeed.
1470   connection()->SetServerReachable();
1471   PumpLoopFor(2 * delta);
1472   ASSERT_EQ(1, success_counter.times_called());
1473   ASSERT_FALSE(scheduler()->IsBackingOff());
1474 }
1475
1476 }  // namespace syncer