sync/engine/sync_scheduler_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/bind.h"
   6 #include "base/callback.h"
   7 #include "base/compiler_specific.h"
   8 #include "base/memory/weak_ptr.h"
   9 #include "base/message_loop/message_loop.h"
  10 #include "base/test/test_timeouts.h"
  11 #include "sync/engine/backoff_delay_provider.h"
  12 #include "sync/engine/sync_scheduler_impl.h"
  13 #include "sync/engine/syncer.h"
  14 #include "sync/internal_api/public/base/cancelation_signal.h"
  15 #include "sync/internal_api/public/base/model_type_test_util.h"
  16 #include "sync/sessions/test_util.h"
  17 #include "sync/test/callback_counter.h"
  18 #include "sync/test/engine/fake_model_worker.h"
  19 #include "sync/test/engine/mock_connection_manager.h"
  20 #include "sync/test/engine/mock_nudge_handler.h"
  21 #include "sync/test/engine/test_directory_setter_upper.h"
  22 #include "sync/test/mock_invalidation.h"
  23 #include "sync/util/extensions_activity.h"
  24 #include "testing/gmock/include/gmock/gmock.h"
  25 #include "testing/gtest/include/gtest/gtest.h"
  26
  27 using base::TimeDelta;
  28 using base::TimeTicks;
  29 using testing::_;
  30 using testing::AtLeast;
  31 using testing::DoAll;
  32 using testing::Invoke;
  33 using testing::Mock;
  34 using testing::Return;
  35 using testing::WithArg;
  36 using testing::WithArgs;
  37 using testing::WithoutArgs;
  38
  39 namespace syncer {
  40 using sessions::SyncSession;
  41 using sessions::SyncSessionContext;
  42 using sync_pb::GetUpdatesCallerInfo;
  43
  44 class MockSyncer : public Syncer {
  45  public:
  46   MockSyncer();
  47   MOCK_METHOD3(NormalSyncShare, bool(ModelTypeSet,
  48                                      const sessions::NudgeTracker&,
  49                                      sessions::SyncSession*));
  50   MOCK_METHOD3(ConfigureSyncShare,
  51                bool(ModelTypeSet,
  52                     sync_pb::GetUpdatesCallerInfo::GetUpdatesSource,
  53                     SyncSession*));
  54   MOCK_METHOD2(PollSyncShare, bool(ModelTypeSet, sessions::SyncSession*));
  55 };
  56
  57 MockSyncer::MockSyncer()
  58   : Syncer(NULL) {}
  59
  60 typedef std::vector<TimeTicks> SyncShareTimes;
  61
  62 void QuitLoopNow() {
  63   // We use QuitNow() instead of Quit() as the latter may get stalled
  64   // indefinitely in the presence of repeated timers with low delays
  65   // and a slow test (e.g., ThrottlingDoesThrottle [which has a poll
  66   // delay of 5ms] run under TSAN on the trybots).
  67   base::MessageLoop::current()->QuitNow();
  68 }
  69
  70 void RunLoop() {
  71   base::MessageLoop::current()->Run();
  72 }
  73
  74 void PumpLoop() {
  75   // Do it this way instead of RunAllPending to pump loop exactly once
  76   // (necessary in the presence of timers; see comment in
  77   // QuitLoopNow).
  78   base::MessageLoop::current()->PostTask(FROM_HERE, base::Bind(&QuitLoopNow));
  79   RunLoop();
  80 }
  81
  82 void PumpLoopFor(base::TimeDelta time) {
  83   // Allow the loop to run for the specified amount of time.
  84   base::MessageLoop::current()->PostDelayedTask(
  85       FROM_HERE, base::Bind(&QuitLoopNow), time);
  86   RunLoop();
  87 }
  88
  89 ModelSafeRoutingInfo TypesToRoutingInfo(ModelTypeSet types) {
  90   ModelSafeRoutingInfo routes;
  91   for (ModelTypeSet::Iterator iter = types.First(); iter.Good(); iter.Inc()) {
  92     routes[iter.Get()] = GROUP_PASSIVE;
  93   }
  94   return routes;
  95 }
  96
  97
  98 static const size_t kMinNumSamples = 5;
  99
 100 // Test harness for the SyncScheduler.  Test the delays and backoff timers used
 101 // in response to various events.
 102 //
 103 // These tests execute in real time with real timers.  We try to keep the
 104 // delays short, but there is a limit to how short we can make them.  The
 105 // timers on some platforms (ie. Windows) have a timer resolution greater than
 106 // 1ms.  Using 1ms delays may result in test flakiness.
 107 //
 108 // See crbug.com/402212 for more info.
 109 class SyncSchedulerTest : public testing::Test {
 110  public:
 111   SyncSchedulerTest() : syncer_(NULL), delay_(NULL), weak_ptr_factory_(this) {}
 112
 113   class MockDelayProvider : public BackoffDelayProvider {
 114    public:
 115     MockDelayProvider() : BackoffDelayProvider(
 116         TimeDelta::FromSeconds(kInitialBackoffRetrySeconds),
 117         TimeDelta::FromSeconds(kInitialBackoffImmediateRetrySeconds)) {
 118     }
 119
 120     MOCK_METHOD1(GetDelay, TimeDelta(const TimeDelta&));
 121   };
 122
 123   void SetUp() override {
 124     dir_maker_.SetUp();
 125     syncer_ = new testing::StrictMock<MockSyncer>();
 126     delay_ = NULL;
 127     extensions_activity_ = new ExtensionsActivity();
 128
 129     routing_info_[THEMES] = GROUP_UI;
 130     routing_info_[TYPED_URLS] = GROUP_DB;
 131     routing_info_[THEMES] = GROUP_UI;
 132     routing_info_[NIGORI] = GROUP_PASSIVE;
 133
 134     workers_.clear();
 135     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_UI)));
 136     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_DB)));
 137     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_PASSIVE)));
 138
 139     connection_.reset(new MockConnectionManager(directory(),
 140                                                 &cancelation_signal_));
 141     connection_->SetServerReachable();
 142
 143     model_type_registry_.reset(
 144         new ModelTypeRegistry(workers_, directory(), &mock_nudge_handler_));
 145
 146     context_.reset(new SyncSessionContext(
 147             connection_.get(), directory(),
 148             extensions_activity_.get(),
 149             std::vector<SyncEngineEventListener*>(), NULL,
 150             model_type_registry_.get(),
 151             true,  // enable keystore encryption
 152             false,  // force enable pre-commit GU avoidance
 153             "fake_invalidator_client_id"));
 154     context_->SetRoutingInfo(routing_info_);
 155     context_->set_notifications_enabled(true);
 156     context_->set_account_name("Test");
 157     scheduler_.reset(
 158         new SyncSchedulerImpl("TestSyncScheduler",
 159             BackoffDelayProvider::FromDefaults(),
 160             context(),
 161             syncer_));
 162     scheduler_->SetDefaultNudgeDelay(default_delay());
 163   }
 164
 165   SyncSchedulerImpl* scheduler() { return scheduler_.get(); }
 166   const ModelSafeRoutingInfo& routing_info() { return routing_info_; }
 167   MockSyncer* syncer() { return syncer_; }
 168   MockDelayProvider* delay() { return delay_; }
 169   MockConnectionManager* connection() { return connection_.get(); }
 170   TimeDelta default_delay() { return TimeDelta::FromSeconds(0); }
 171   TimeDelta timeout() {
 172     return TestTimeouts::action_timeout();
 173   }
 174
 175   void TearDown() override {
 176     PumpLoop();
 177     scheduler_.reset();
 178     PumpLoop();
 179     dir_maker_.TearDown();
 180   }
 181
 182   void AnalyzePollRun(const SyncShareTimes& times, size_t min_num_samples,
 183       const TimeTicks& optimal_start, const TimeDelta& poll_interval) {
 184     EXPECT_GE(times.size(), min_num_samples);
 185     for (size_t i = 0; i < times.size(); i++) {
 186       SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
 187       TimeTicks optimal_next_sync = optimal_start + poll_interval * i;
 188       EXPECT_GE(times[i], optimal_next_sync);
 189     }
 190   }
 191
 192   void DoQuitLoopNow() {
 193     QuitLoopNow();
 194   }
 195
 196   void StartSyncScheduler(SyncScheduler::Mode mode) {
 197     scheduler()->Start(mode);
 198   }
 199
 200   // This stops the scheduler synchronously.
 201   void StopSyncScheduler() {
 202     base::MessageLoop::current()->PostTask(
 203         FROM_HERE,
 204         base::Bind(&SyncSchedulerTest::DoQuitLoopNow,
 205                    weak_ptr_factory_.GetWeakPtr()));
 206     RunLoop();
 207   }
 208
 209   bool RunAndGetBackoff() {
 210     ModelTypeSet nudge_types(THEMES);
 211     StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 212
 213     scheduler()->ScheduleLocalNudge(nudge_types, FROM_HERE);
 214     RunLoop();
 215
 216     return scheduler()->IsBackingOff();
 217   }
 218
 219   void UseMockDelayProvider() {
 220     delay_ = new MockDelayProvider();
 221     scheduler_->delay_provider_.reset(delay_);
 222   }
 223
 224   SyncSessionContext* context() { return context_.get(); }
 225
 226   ModelTypeSet GetThrottledTypes() {
 227     return scheduler_->nudge_tracker_.GetThrottledTypes();
 228   }
 229
 230   base::TimeDelta GetRetryTimerDelay() {
 231     EXPECT_TRUE(scheduler_->retry_timer_.IsRunning());
 232     return scheduler_->retry_timer_.GetCurrentDelay();
 233   }
 234
 235   static scoped_ptr<InvalidationInterface> BuildInvalidation(
 236       int64 version,
 237       const std::string& payload) {
 238     return MockInvalidation::Build(version, payload);
 239   }
 240
 241  private:
 242   syncable::Directory* directory() {
 243     return dir_maker_.directory();
 244   }
 245
 246   base::MessageLoop loop_;
 247   TestDirectorySetterUpper dir_maker_;
 248   CancelationSignal cancelation_signal_;
 249   scoped_ptr<MockConnectionManager> connection_;
 250   scoped_ptr<ModelTypeRegistry> model_type_registry_;
 251   scoped_ptr<SyncSessionContext> context_;
 252   scoped_ptr<SyncSchedulerImpl> scheduler_;
 253   MockNudgeHandler mock_nudge_handler_;
 254   MockSyncer* syncer_;
 255   MockDelayProvider* delay_;
 256   std::vector<scoped_refptr<ModelSafeWorker> > workers_;
 257   scoped_refptr<ExtensionsActivity> extensions_activity_;
 258   ModelSafeRoutingInfo routing_info_;
 259   base::WeakPtrFactory<SyncSchedulerTest> weak_ptr_factory_;
 260 };
 261
 262 void RecordSyncShareImpl(SyncShareTimes* times) {
 263   times->push_back(TimeTicks::Now());
 264 }
 265
 266 ACTION_P(RecordSyncShare, times) {
 267   RecordSyncShareImpl(times);
 268   if (base::MessageLoop::current()->is_running())
 269     QuitLoopNow();
 270   return true;
 271 }
 272
 273 ACTION_P2(RecordSyncShareMultiple, times, quit_after) {
 274   RecordSyncShareImpl(times);
 275   EXPECT_LE(times->size(), quit_after);
 276   if (times->size() >= quit_after &&
 277       base::MessageLoop::current()->is_running()) {
 278     QuitLoopNow();
 279   }
 280   return true;
 281 }
 282
 283 ACTION_P(StopScheduler, scheduler) {
 284   scheduler->Stop();
 285 }
 286
 287 ACTION(AddFailureAndQuitLoopNow) {
 288   ADD_FAILURE();
 289   QuitLoopNow();
 290   return true;
 291 }
 292
 293 ACTION(QuitLoopNowAction) {
 294   QuitLoopNow();
 295   return true;
 296 }
 297
 298 // Test nudge scheduling.
 299 TEST_F(SyncSchedulerTest, Nudge) {
 300   SyncShareTimes times;
 301   ModelTypeSet model_types(THEMES);
 302
 303   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 304       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 305                       RecordSyncShare(&times)))
 306       .RetiresOnSaturation();
 307
 308   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 309
 310   scheduler()->ScheduleLocalNudge(model_types, FROM_HERE);
 311   RunLoop();
 312
 313   Mock::VerifyAndClearExpectations(syncer());
 314
 315   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 316   SyncShareTimes times2;
 317   model_types.Remove(THEMES);
 318   model_types.Put(TYPED_URLS);
 319   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 320       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 321                       RecordSyncShare(&times2)));
 322   scheduler()->ScheduleLocalNudge(model_types, FROM_HERE);
 323   RunLoop();
 324 }
 325
 326 // Make sure a regular config command is scheduled fine in the absence of any
 327 // errors.
 328 TEST_F(SyncSchedulerTest, Config) {
 329   SyncShareTimes times;
 330   const ModelTypeSet model_types(THEMES);
 331
 332   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 333       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 334                       RecordSyncShare(&times)));
 335
 336   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 337
 338   CallbackCounter ready_counter;
 339   CallbackCounter retry_counter;
 340   ConfigurationParams params(
 341       GetUpdatesCallerInfo::RECONFIGURATION,
 342       model_types,
 343       TypesToRoutingInfo(model_types),
 344       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 345       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 346   scheduler()->ScheduleConfiguration(params);
 347   PumpLoop();
 348   ASSERT_EQ(1, ready_counter.times_called());
 349   ASSERT_EQ(0, retry_counter.times_called());
 350 }
 351
 352 // Simulate a failure and make sure the config request is retried.
 353 TEST_F(SyncSchedulerTest, ConfigWithBackingOff) {
 354   UseMockDelayProvider();
 355   EXPECT_CALL(*delay(), GetDelay(_))
 356       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(20)));
 357   SyncShareTimes times;
 358   const ModelTypeSet model_types(THEMES);
 359
 360   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 361
 362   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 363       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 364                       RecordSyncShare(&times)))
 365       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 366                       RecordSyncShare(&times)));
 367
 368   CallbackCounter ready_counter;
 369   CallbackCounter retry_counter;
 370   ConfigurationParams params(
 371       GetUpdatesCallerInfo::RECONFIGURATION,
 372       model_types,
 373       TypesToRoutingInfo(model_types),
 374       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 375       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 376   scheduler()->ScheduleConfiguration(params);
 377   RunLoop();
 378   ASSERT_EQ(0, ready_counter.times_called());
 379   ASSERT_EQ(1, retry_counter.times_called());
 380
 381   // RunLoop() will trigger TryCanaryJob which will retry configuration.
 382   // Since retry_task was already called it shouldn't be called again.
 383   RunLoop();
 384   ASSERT_EQ(0, ready_counter.times_called());
 385   ASSERT_EQ(1, retry_counter.times_called());
 386
 387   Mock::VerifyAndClearExpectations(syncer());
 388
 389   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 390       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 391                       RecordSyncShare(&times)));
 392   RunLoop();
 393
 394   ASSERT_EQ(1, ready_counter.times_called());
 395 }
 396
 397 // Simuilate SyncSchedulerImpl::Stop being called in the middle of Configure.
 398 // This can happen if server returns NOT_MY_BIRTHDAY.
 399 TEST_F(SyncSchedulerTest, ConfigWithStop) {
 400   UseMockDelayProvider();
 401   EXPECT_CALL(*delay(), GetDelay(_))
 402       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(20)));
 403   SyncShareTimes times;
 404   const ModelTypeSet model_types(THEMES);
 405
 406   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 407
 408   // Make ConfigureSyncShare call scheduler->Stop(). It is not supposed to call
 409   // retry_task or dereference configuration params.
 410   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 411       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 412                       StopScheduler(scheduler()),
 413                       RecordSyncShare(&times)));
 414
 415   CallbackCounter ready_counter;
 416   CallbackCounter retry_counter;
 417   ConfigurationParams params(
 418       GetUpdatesCallerInfo::RECONFIGURATION,
 419       model_types,
 420       TypesToRoutingInfo(model_types),
 421       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 422       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 423   scheduler()->ScheduleConfiguration(params);
 424   PumpLoop();
 425   ASSERT_EQ(0, ready_counter.times_called());
 426   ASSERT_EQ(0, retry_counter.times_called());
 427 }
 428
 429 // Issue a nudge when the config has failed. Make sure both the config and
 430 // nudge are executed.
 431 TEST_F(SyncSchedulerTest, NudgeWithConfigWithBackingOff) {
 432   const ModelTypeSet model_types(THEMES);
 433   UseMockDelayProvider();
 434   EXPECT_CALL(*delay(), GetDelay(_))
 435       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(50)));
 436   SyncShareTimes times;
 437
 438   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 439
 440   // Request a configure and make sure it fails.
 441   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 442       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 443                       RecordSyncShare(&times)));
 444   CallbackCounter ready_counter;
 445   CallbackCounter retry_counter;
 446   ConfigurationParams params(
 447       GetUpdatesCallerInfo::RECONFIGURATION,
 448       model_types,
 449       TypesToRoutingInfo(model_types),
 450       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 451       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 452   scheduler()->ScheduleConfiguration(params);
 453   RunLoop();
 454   ASSERT_EQ(0, ready_counter.times_called());
 455   ASSERT_EQ(1, retry_counter.times_called());
 456   Mock::VerifyAndClearExpectations(syncer());
 457
 458   // Ask for a nudge while dealing with repeated configure failure.
 459   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 460       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 461                       RecordSyncShare(&times)));
 462   scheduler()->ScheduleLocalNudge(model_types, FROM_HERE);
 463   RunLoop();
 464   // Note that we're not RunLoop()ing for the NUDGE we just scheduled, but
 465   // for the first retry attempt from the config job (after
 466   // waiting ~+/- 50ms).
 467   Mock::VerifyAndClearExpectations(syncer());
 468   ASSERT_EQ(0, ready_counter.times_called());
 469
 470   // Let the next configure retry succeed.
 471   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 472       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 473                       RecordSyncShare(&times)));
 474   RunLoop();
 475
 476   // Now change the mode so nudge can execute.
 477   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 478       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 479                       RecordSyncShare(&times)));
 480   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 481   PumpLoop();
 482 }
 483
 484 // Test that nudges are coalesced.
 485 TEST_F(SyncSchedulerTest, NudgeCoalescing) {
 486   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 487
 488   SyncShareTimes times;
 489   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 490       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 491                       RecordSyncShare(&times)));
 492   const ModelTypeSet types1(THEMES), types2(TYPED_URLS), types3(THEMES);
 493   TimeTicks optimal_time = TimeTicks::Now() + default_delay();
 494   scheduler()->ScheduleLocalNudge(types1, FROM_HERE);
 495   scheduler()->ScheduleLocalNudge(types2, FROM_HERE);
 496   RunLoop();
 497
 498   ASSERT_EQ(1U, times.size());
 499   EXPECT_GE(times[0], optimal_time);
 500
 501   Mock::VerifyAndClearExpectations(syncer());
 502
 503   SyncShareTimes times2;
 504   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 505       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 506                       RecordSyncShare(&times2)));
 507   scheduler()->ScheduleLocalNudge(types3, FROM_HERE);
 508   RunLoop();
 509 }
 510
 511 // Test that nudges are coalesced.
 512 TEST_F(SyncSchedulerTest, NudgeCoalescingWithDifferentTimings) {
 513   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 514
 515   SyncShareTimes times;
 516   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 517       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 518                       RecordSyncShare(&times)));
 519   ModelTypeSet types1(THEMES), types2(TYPED_URLS), types3;
 520
 521   // Create a huge time delay.
 522   TimeDelta delay = TimeDelta::FromDays(1);
 523
 524   std::map<ModelType, TimeDelta> delay_map;
 525   delay_map[types1.First().Get()] = delay;
 526   scheduler()->OnReceivedCustomNudgeDelays(delay_map);
 527   scheduler()->ScheduleLocalNudge(types1, FROM_HERE);
 528   scheduler()->ScheduleLocalNudge(types2, FROM_HERE);
 529
 530   TimeTicks min_time = TimeTicks::Now();
 531   TimeTicks max_time = TimeTicks::Now() + delay;
 532
 533   RunLoop();
 534   Mock::VerifyAndClearExpectations(syncer());
 535
 536   // Make sure the sync happened at the right time.
 537   ASSERT_EQ(1U, times.size());
 538   EXPECT_GE(times[0], min_time);
 539   EXPECT_LE(times[0], max_time);
 540 }
 541
 542 // Test nudge scheduling.
 543 TEST_F(SyncSchedulerTest, NudgeWithStates) {
 544   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 545
 546   SyncShareTimes times1;
 547   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 548       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 549                       RecordSyncShare(&times1)))
 550       .RetiresOnSaturation();
 551   scheduler()->ScheduleInvalidationNudge(
 552       THEMES, BuildInvalidation(10, "test"), FROM_HERE);
 553   RunLoop();
 554
 555   Mock::VerifyAndClearExpectations(syncer());
 556
 557   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 558   SyncShareTimes times2;
 559   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 560       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 561                       RecordSyncShare(&times2)));
 562   scheduler()->ScheduleInvalidationNudge(
 563       TYPED_URLS, BuildInvalidation(10, "test2"), FROM_HERE);
 564   RunLoop();
 565 }
 566
 567 // Test that polling works as expected.
 568 TEST_F(SyncSchedulerTest, Polling) {
 569   SyncShareTimes times;
 570   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 571   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 572       .WillRepeatedly(
 573           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 574                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 575
 576   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
 577
 578   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 579   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 580
 581   // Run again to wait for polling.
 582   RunLoop();
 583
 584   StopSyncScheduler();
 585   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 586 }
 587
 588 // Test that the short poll interval is used.
 589 TEST_F(SyncSchedulerTest, PollNotificationsDisabled) {
 590   SyncShareTimes times;
 591   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 592   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 593       .WillRepeatedly(
 594           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 595                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 596
 597   scheduler()->OnReceivedShortPollIntervalUpdate(poll_interval);
 598   scheduler()->SetNotificationsEnabled(false);
 599
 600   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 601   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 602
 603   // Run again to wait for polling.
 604   RunLoop();
 605
 606   StopSyncScheduler();
 607   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 608 }
 609
 610 // Test that polling intervals are updated when needed.
 611 TEST_F(SyncSchedulerTest, PollIntervalUpdate) {
 612   SyncShareTimes times;
 613   TimeDelta poll1(TimeDelta::FromMilliseconds(120));
 614   TimeDelta poll2(TimeDelta::FromMilliseconds(30));
 615   scheduler()->OnReceivedLongPollIntervalUpdate(poll1);
 616   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 617       .WillOnce(DoAll(
 618           WithArgs<0,1>(
 619               sessions::test_util::SimulatePollIntervalUpdate(poll2)),
 620           Return(true)))
 621       .WillRepeatedly(
 622           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 623                 WithArg<1>(
 624                     RecordSyncShareMultiple(&times, kMinNumSamples))));
 625
 626   TimeTicks optimal_start = TimeTicks::Now() + poll1 + poll2;
 627   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 628
 629   // Run again to wait for polling.
 630   RunLoop();
 631
 632   StopSyncScheduler();
 633   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll2);
 634 }
 635
 636 // Test that no syncing occurs when throttled.
 637 TEST_F(SyncSchedulerTest, ThrottlingDoesThrottle) {
 638   const ModelTypeSet types(THEMES);
 639   TimeDelta poll(TimeDelta::FromMilliseconds(20));
 640   TimeDelta throttle(TimeDelta::FromMinutes(10));
 641   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 642
 643   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 644       .WillOnce(DoAll(
 645           WithArg<2>(sessions::test_util::SimulateThrottled(throttle)),
 646           Return(true)))
 647       .WillRepeatedly(AddFailureAndQuitLoopNow());
 648
 649   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 650
 651   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
 652   PumpLoop();
 653
 654   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 655
 656   CallbackCounter ready_counter;
 657   CallbackCounter retry_counter;
 658   ConfigurationParams params(
 659       GetUpdatesCallerInfo::RECONFIGURATION,
 660       types,
 661       TypesToRoutingInfo(types),
 662       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 663       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 664   scheduler()->ScheduleConfiguration(params);
 665   PumpLoop();
 666   ASSERT_EQ(0, ready_counter.times_called());
 667   ASSERT_EQ(1, retry_counter.times_called());
 668
 669 }
 670
 671 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromPoll) {
 672   SyncShareTimes times;
 673   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 674   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 675   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 676
 677   ::testing::InSequence seq;
 678   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 679       .WillOnce(DoAll(
 680           WithArg<1>(sessions::test_util::SimulateThrottled(throttle1)),
 681           Return(true)))
 682       .RetiresOnSaturation();
 683   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 684       .WillRepeatedly(
 685           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 686                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 687
 688   TimeTicks optimal_start = TimeTicks::Now() + poll + throttle1;
 689   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 690
 691   // Run again to wait for polling.
 692   RunLoop();
 693
 694   StopSyncScheduler();
 695   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll);
 696 }
 697
 698 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromNudge) {
 699   SyncShareTimes times;
 700   TimeDelta poll(TimeDelta::FromDays(1));
 701   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 702   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 703
 704   ::testing::InSequence seq;
 705   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 706       .WillOnce(DoAll(
 707           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 708           Return(true)))
 709       .RetiresOnSaturation();
 710   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 711       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 712                       QuitLoopNowAction()));
 713
 714   const ModelTypeSet types(THEMES);
 715   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 716   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
 717
 718   PumpLoop(); // To get PerformDelayedNudge called.
 719   PumpLoop(); // To get TrySyncSessionJob called
 720   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 721   RunLoop();
 722   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 723
 724   StopSyncScheduler();
 725 }
 726
 727 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromConfigure) {
 728   SyncShareTimes times;
 729   TimeDelta poll(TimeDelta::FromDays(1));
 730   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 731   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 732
 733   ::testing::InSequence seq;
 734   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 735       .WillOnce(DoAll(
 736           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 737           Return(true)))
 738       .RetiresOnSaturation();
 739   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 740       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 741                       QuitLoopNowAction()));
 742
 743   const ModelTypeSet types(THEMES);
 744   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 745
 746   CallbackCounter ready_counter;
 747   CallbackCounter retry_counter;
 748   ConfigurationParams params(
 749       GetUpdatesCallerInfo::RECONFIGURATION,
 750       types,
 751       TypesToRoutingInfo(types),
 752       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 753       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 754   scheduler()->ScheduleConfiguration(params);
 755   PumpLoop();
 756   EXPECT_EQ(0, ready_counter.times_called());
 757   EXPECT_EQ(1, retry_counter.times_called());
 758   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 759
 760   RunLoop();
 761   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 762
 763   StopSyncScheduler();
 764 }
 765
 766 TEST_F(SyncSchedulerTest, TypeThrottlingBlocksNudge) {
 767   UseMockDelayProvider();
 768   EXPECT_CALL(*delay(), GetDelay(_))
 769       .WillRepeatedly(Return(default_delay()));
 770
 771   TimeDelta poll(TimeDelta::FromDays(1));
 772   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 773   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 774
 775   const ModelTypeSet types(THEMES);
 776
 777   ::testing::InSequence seq;
 778   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 779       .WillOnce(DoAll(
 780           WithArg<2>(
 781               sessions::test_util::SimulateTypesThrottled(types, throttle1)),
 782           Return(true)))
 783       .RetiresOnSaturation();
 784
 785   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 786   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
 787   PumpLoop(); // To get PerformDelayedNudge called.
 788   PumpLoop(); // To get TrySyncSessionJob called
 789   EXPECT_TRUE(GetThrottledTypes().HasAll(types));
 790
 791   // This won't cause a sync cycle because the types are throttled.
 792   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
 793   PumpLoop();
 794
 795   StopSyncScheduler();
 796 }
 797
 798 TEST_F(SyncSchedulerTest, TypeThrottlingDoesBlockOtherSources) {
 799   UseMockDelayProvider();
 800   EXPECT_CALL(*delay(), GetDelay(_))
 801       .WillRepeatedly(Return(default_delay()));
 802
 803   SyncShareTimes times;
 804   TimeDelta poll(TimeDelta::FromDays(1));
 805   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 806   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 807
 808   const ModelTypeSet throttled_types(THEMES);
 809   const ModelTypeSet unthrottled_types(PREFERENCES);
 810
 811   ::testing::InSequence seq;
 812   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 813       .WillOnce(DoAll(
 814           WithArg<2>(
 815               sessions::test_util::SimulateTypesThrottled(
 816                   throttled_types, throttle1)),
 817           Return(true)))
 818       .RetiresOnSaturation();
 819
 820   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 821   scheduler()->ScheduleLocalNudge(throttled_types, FROM_HERE);
 822   PumpLoop(); // To get PerformDelayedNudge called.
 823   PumpLoop(); // To get TrySyncSessionJob called
 824   EXPECT_TRUE(GetThrottledTypes().HasAll(throttled_types));
 825
 826   // Ignore invalidations for throttled types.
 827   scheduler()->ScheduleInvalidationNudge(
 828       THEMES, BuildInvalidation(10, "test"), FROM_HERE);
 829   PumpLoop();
 830
 831   // Ignore refresh requests for throttled types.
 832   scheduler()->ScheduleLocalRefreshRequest(throttled_types, FROM_HERE);
 833   PumpLoop();
 834
 835   Mock::VerifyAndClearExpectations(syncer());
 836
 837   // Local nudges for non-throttled types will trigger a sync.
 838   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 839       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 840                             RecordSyncShare(&times)));
 841   scheduler()->ScheduleLocalNudge(unthrottled_types, FROM_HERE);
 842   RunLoop();
 843   Mock::VerifyAndClearExpectations(syncer());
 844
 845   StopSyncScheduler();
 846 }
 847
 848 // Test nudges / polls don't run in config mode and config tasks do.
 849 TEST_F(SyncSchedulerTest, ConfigurationMode) {
 850   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 851   SyncShareTimes times;
 852   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 853
 854   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 855
 856   const ModelTypeSet nudge_types(TYPED_URLS);
 857   scheduler()->ScheduleLocalNudge(nudge_types, FROM_HERE);
 858   scheduler()->ScheduleLocalNudge(nudge_types, FROM_HERE);
 859
 860   const ModelTypeSet config_types(THEMES);
 861
 862   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 863       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 864                       RecordSyncShare(&times)))
 865       .RetiresOnSaturation();
 866   CallbackCounter ready_counter;
 867   CallbackCounter retry_counter;
 868   ConfigurationParams params(
 869       GetUpdatesCallerInfo::RECONFIGURATION,
 870       config_types,
 871       TypesToRoutingInfo(config_types),
 872       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 873       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 874   scheduler()->ScheduleConfiguration(params);
 875   RunLoop();
 876   ASSERT_EQ(1, ready_counter.times_called());
 877   ASSERT_EQ(0, retry_counter.times_called());
 878
 879   Mock::VerifyAndClearExpectations(syncer());
 880
 881   // Switch to NORMAL_MODE to ensure NUDGES were properly saved and run.
 882   scheduler()->OnReceivedLongPollIntervalUpdate(TimeDelta::FromDays(1));
 883   SyncShareTimes times2;
 884   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 885       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 886                       RecordSyncShare(&times2)));
 887
 888   // TODO(tim): Figure out how to remove this dangerous need to reset
 889   // routing info between mode switches.
 890   context()->SetRoutingInfo(routing_info());
 891   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 892
 893   RunLoop();
 894   Mock::VerifyAndClearExpectations(syncer());
 895 }
 896
 897 class BackoffTriggersSyncSchedulerTest : public SyncSchedulerTest {
 898   void SetUp() override {
 899     SyncSchedulerTest::SetUp();
 900     UseMockDelayProvider();
 901     EXPECT_CALL(*delay(), GetDelay(_))
 902         .WillRepeatedly(Return(TimeDelta::FromMilliseconds(10)));
 903   }
 904
 905   void TearDown() override {
 906     StopSyncScheduler();
 907     SyncSchedulerTest::TearDown();
 908   }
 909 };
 910
 911 // Have the sycner fail during commit.  Expect that the scheduler enters
 912 // backoff.
 913 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnce) {
 914   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 915       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
 916                       QuitLoopNowAction()));
 917   EXPECT_TRUE(RunAndGetBackoff());
 918 }
 919
 920 // Have the syncer fail during download updates and succeed on the first
 921 // retry.  Expect that this clears the backoff state.
 922 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadOnceThenSucceed) {
 923   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 924       .WillOnce(DoAll(
 925           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 926           Return(true)))
 927       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 928                       QuitLoopNowAction()));
 929   EXPECT_FALSE(RunAndGetBackoff());
 930 }
 931
 932 // Have the syncer fail during commit and succeed on the first retry.  Expect
 933 // that this clears the backoff state.
 934 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnceThenSucceed) {
 935   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 936       .WillOnce(DoAll(
 937           Invoke(sessions::test_util::SimulateCommitFailed),
 938           Return(true)))
 939       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 940                       QuitLoopNowAction()));
 941   EXPECT_FALSE(RunAndGetBackoff());
 942 }
 943
 944 // Have the syncer fail to download updates and fail again on the retry.
 945 // Expect this will leave the scheduler in backoff.
 946 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadTwice) {
 947   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 948       .WillOnce(DoAll(
 949           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 950           Return(true)))
 951       .WillRepeatedly(DoAll(
 952               Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 953               QuitLoopNowAction()));
 954   EXPECT_TRUE(RunAndGetBackoff());
 955 }
 956
 957 // Have the syncer fail to get the encryption key yet succeed in downloading
 958 // updates. Expect this will leave the scheduler in backoff.
 959 TEST_F(BackoffTriggersSyncSchedulerTest, FailGetEncryptionKey) {
 960   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 961       .WillOnce(DoAll(
 962           Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
 963           Return(true)))
 964       .WillRepeatedly(DoAll(
 965               Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
 966               QuitLoopNowAction()));
 967   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 968
 969   ModelTypeSet types(THEMES);
 970   CallbackCounter ready_counter;
 971   CallbackCounter retry_counter;
 972   ConfigurationParams params(
 973       GetUpdatesCallerInfo::RECONFIGURATION,
 974       types,
 975       TypesToRoutingInfo(types),
 976       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 977       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 978   scheduler()->ScheduleConfiguration(params);
 979   RunLoop();
 980
 981   EXPECT_TRUE(scheduler()->IsBackingOff());
 982 }
 983
 984 // Test that no polls or extraneous nudges occur when in backoff.
 985 TEST_F(SyncSchedulerTest, BackoffDropsJobs) {
 986   SyncShareTimes times;
 987   TimeDelta poll(TimeDelta::FromMilliseconds(10));
 988   const ModelTypeSet types(THEMES);
 989   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 990   UseMockDelayProvider();
 991
 992   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 993       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
 994                       RecordSyncShareMultiple(&times, 1U)));
 995   EXPECT_CALL(*delay(), GetDelay(_)).
 996       WillRepeatedly(Return(TimeDelta::FromDays(1)));
 997
 998   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 999
1000   // This nudge should fail and put us into backoff.  Thanks to our mock
1001   // GetDelay() setup above, this will be a long backoff.
1002   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
1003   RunLoop();
1004
1005   // From this point forward, no SyncShare functions should be invoked.
1006   Mock::VerifyAndClearExpectations(syncer());
1007
1008   // Wait a while (10x poll interval) so a few poll jobs will be attempted.
1009   PumpLoopFor(poll * 10);
1010
1011   // Try (and fail) to schedule a nudge.
1012   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
1013
1014   Mock::VerifyAndClearExpectations(syncer());
1015   Mock::VerifyAndClearExpectations(delay());
1016
1017   EXPECT_CALL(*delay(), GetDelay(_)).Times(0);
1018
1019   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1020
1021   CallbackCounter ready_counter;
1022   CallbackCounter retry_counter;
1023   ConfigurationParams params(
1024       GetUpdatesCallerInfo::RECONFIGURATION,
1025       types,
1026       TypesToRoutingInfo(types),
1027       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1028       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1029   scheduler()->ScheduleConfiguration(params);
1030   PumpLoop();
1031   ASSERT_EQ(0, ready_counter.times_called());
1032   ASSERT_EQ(1, retry_counter.times_called());
1033
1034 }
1035
1036 // Test that backoff is shaping traffic properly with consecutive errors.
1037 TEST_F(SyncSchedulerTest, BackoffElevation) {
1038   SyncShareTimes times;
1039   UseMockDelayProvider();
1040
1041   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_)).Times(kMinNumSamples)
1042       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1043           RecordSyncShareMultiple(&times, kMinNumSamples)));
1044
1045   const TimeDelta first = TimeDelta::FromSeconds(kInitialBackoffRetrySeconds);
1046   const TimeDelta second = TimeDelta::FromMilliseconds(20);
1047   const TimeDelta third = TimeDelta::FromMilliseconds(30);
1048   const TimeDelta fourth = TimeDelta::FromMilliseconds(40);
1049   const TimeDelta fifth = TimeDelta::FromMilliseconds(50);
1050   const TimeDelta sixth = TimeDelta::FromDays(1);
1051
1052   EXPECT_CALL(*delay(), GetDelay(first)).WillOnce(Return(second))
1053           .RetiresOnSaturation();
1054   EXPECT_CALL(*delay(), GetDelay(second)).WillOnce(Return(third))
1055           .RetiresOnSaturation();
1056   EXPECT_CALL(*delay(), GetDelay(third)).WillOnce(Return(fourth))
1057           .RetiresOnSaturation();
1058   EXPECT_CALL(*delay(), GetDelay(fourth)).WillOnce(Return(fifth))
1059           .RetiresOnSaturation();
1060   EXPECT_CALL(*delay(), GetDelay(fifth)).WillOnce(Return(sixth));
1061
1062   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1063
1064   // Run again with a nudge.
1065   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1066   RunLoop();
1067
1068   ASSERT_EQ(kMinNumSamples, times.size());
1069   EXPECT_GE(times[1] - times[0], second);
1070   EXPECT_GE(times[2] - times[1], third);
1071   EXPECT_GE(times[3] - times[2], fourth);
1072   EXPECT_GE(times[4] - times[3], fifth);
1073 }
1074
1075 // Test that things go back to normal once a retry makes forward progress.
1076 TEST_F(SyncSchedulerTest, BackoffRelief) {
1077   SyncShareTimes times;
1078   const TimeDelta poll(TimeDelta::FromMilliseconds(10));
1079   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1080   UseMockDelayProvider();
1081
1082   const TimeDelta backoff = TimeDelta::FromMilliseconds(10);
1083   EXPECT_CALL(*delay(), GetDelay(_)).WillOnce(Return(backoff));
1084
1085   // Optimal start for the post-backoff poll party.
1086   TimeTicks optimal_start = TimeTicks::Now();
1087   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1088
1089   // Kick off the test with a failed nudge.
1090   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1091       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1092                       RecordSyncShare(&times)));
1093   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1094   RunLoop();
1095   Mock::VerifyAndClearExpectations(syncer());
1096   TimeTicks optimal_job_time = optimal_start;
1097   ASSERT_EQ(1U, times.size());
1098   EXPECT_GE(times[0], optimal_job_time);
1099
1100   // The retry succeeds.
1101   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1102       .WillOnce(DoAll(
1103               Invoke(sessions::test_util::SimulateNormalSuccess),
1104               RecordSyncShare(&times)));
1105   RunLoop();
1106   Mock::VerifyAndClearExpectations(syncer());
1107   optimal_job_time = optimal_job_time + backoff;
1108   ASSERT_EQ(2U, times.size());
1109   EXPECT_GE(times[1], optimal_job_time);
1110
1111   // Now let the Poll timer do its thing.
1112   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1113       .WillRepeatedly(DoAll(
1114               Invoke(sessions::test_util::SimulatePollSuccess),
1115               RecordSyncShareMultiple(&times, kMinNumSamples)));
1116   RunLoop();
1117   Mock::VerifyAndClearExpectations(syncer());
1118   ASSERT_EQ(kMinNumSamples, times.size());
1119   for (size_t i = 2; i < times.size(); i++) {
1120     optimal_job_time = optimal_job_time + poll;
1121     SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
1122     EXPECT_GE(times[i], optimal_job_time);
1123   }
1124
1125   StopSyncScheduler();
1126 }
1127
1128 // Test that poll failures are ignored.  They should have no effect on
1129 // subsequent poll attempts, nor should they trigger a backoff/retry.
1130 TEST_F(SyncSchedulerTest, TransientPollFailure) {
1131   SyncShareTimes times;
1132   const TimeDelta poll_interval(TimeDelta::FromMilliseconds(10));
1133   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
1134   UseMockDelayProvider(); // Will cause test failure if backoff is initiated.
1135
1136   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1137       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollFailed),
1138                       RecordSyncShare(&times)))
1139       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1140                       RecordSyncShare(&times)));
1141
1142   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1143
1144   // Run the unsucessful poll. The failed poll should not trigger backoff.
1145   RunLoop();
1146   EXPECT_FALSE(scheduler()->IsBackingOff());
1147
1148   // Run the successful poll.
1149   RunLoop();
1150   EXPECT_FALSE(scheduler()->IsBackingOff());
1151 }
1152
1153 // Test that starting the syncer thread without a valid connection doesn't
1154 // break things when a connection is detected.
1155 TEST_F(SyncSchedulerTest, StartWhenNotConnected) {
1156   connection()->SetServerNotReachable();
1157   connection()->UpdateConnectionStatus();
1158   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1159     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1160                     Return(true)))
1161     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1162                     Return(true)));
1163   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1164
1165   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1166   // Should save the nudge for until after the server is reachable.
1167   base::MessageLoop::current()->RunUntilIdle();
1168
1169   scheduler()->OnConnectionStatusChange();
1170   connection()->SetServerReachable();
1171   connection()->UpdateConnectionStatus();
1172   base::MessageLoop::current()->RunUntilIdle();
1173 }
1174
1175 TEST_F(SyncSchedulerTest, ServerConnectionChangeDuringBackoff) {
1176   UseMockDelayProvider();
1177   EXPECT_CALL(*delay(), GetDelay(_))
1178       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1179
1180   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1181   connection()->SetServerNotReachable();
1182   connection()->UpdateConnectionStatus();
1183
1184   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1185     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1186                     Return(true)))
1187     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1188                     Return(true)));
1189
1190   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1191   PumpLoop(); // To get PerformDelayedNudge called.
1192   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1193   ASSERT_TRUE(scheduler()->IsBackingOff());
1194
1195   // Before we run the scheduled canary, trigger a server connection change.
1196   scheduler()->OnConnectionStatusChange();
1197   connection()->SetServerReachable();
1198   connection()->UpdateConnectionStatus();
1199   base::MessageLoop::current()->RunUntilIdle();
1200 }
1201
1202 // This was supposed to test the scenario where we receive a nudge while a
1203 // connection change canary is scheduled, but has not run yet.  Since we've made
1204 // the connection change canary synchronous, this is no longer possible.
1205 TEST_F(SyncSchedulerTest, ConnectionChangeCanaryPreemptedByNudge) {
1206   UseMockDelayProvider();
1207   EXPECT_CALL(*delay(), GetDelay(_))
1208       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1209
1210   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1211   connection()->SetServerNotReachable();
1212   connection()->UpdateConnectionStatus();
1213
1214   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1215     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1216                     Return(true)))
1217     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1218                     Return(true)))
1219     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1220                     QuitLoopNowAction()));
1221
1222   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1223
1224   PumpLoop(); // To get PerformDelayedNudge called.
1225   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1226   ASSERT_TRUE(scheduler()->IsBackingOff());
1227
1228   // Before we run the scheduled canary, trigger a server connection change.
1229   scheduler()->OnConnectionStatusChange();
1230   PumpLoop();
1231   connection()->SetServerReachable();
1232   connection()->UpdateConnectionStatus();
1233   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1234   base::MessageLoop::current()->RunUntilIdle();
1235 }
1236
1237 // Tests that we don't crash trying to run two canaries at once if we receive
1238 // extra connection status change notifications.  See crbug.com/190085.
1239 TEST_F(SyncSchedulerTest, DoubleCanaryInConfigure) {
1240   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
1241       .WillRepeatedly(DoAll(
1242               Invoke(sessions::test_util::SimulateConfigureConnectionFailure),
1243               Return(true)));
1244   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1245   connection()->SetServerNotReachable();
1246   connection()->UpdateConnectionStatus();
1247
1248   ModelTypeSet model_types(THEMES);
1249   CallbackCounter ready_counter;
1250   CallbackCounter retry_counter;
1251   ConfigurationParams params(
1252       GetUpdatesCallerInfo::RECONFIGURATION,
1253       model_types,
1254       TypesToRoutingInfo(model_types),
1255       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1256       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1257   scheduler()->ScheduleConfiguration(params);
1258
1259   scheduler()->OnConnectionStatusChange();
1260   scheduler()->OnConnectionStatusChange();
1261
1262   PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
1263 }
1264
1265 TEST_F(SyncSchedulerTest, PollFromCanaryAfterAuthError) {
1266   SyncShareTimes times;
1267   TimeDelta poll(TimeDelta::FromMilliseconds(15));
1268   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1269
1270   ::testing::InSequence seq;
1271   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1272       .WillRepeatedly(
1273           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1274                 RecordSyncShareMultiple(&times, kMinNumSamples)));
1275
1276   connection()->SetServerStatus(HttpResponse::SYNC_AUTH_ERROR);
1277   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1278
1279   // Run to wait for polling.
1280   RunLoop();
1281
1282   // Normally OnCredentialsUpdated calls TryCanaryJob that doesn't run Poll,
1283   // but after poll finished with auth error from poll timer it should retry
1284   // poll once more
1285   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1286       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1287                       RecordSyncShare(&times)));
1288   scheduler()->OnCredentialsUpdated();
1289   connection()->SetServerStatus(HttpResponse::SERVER_CONNECTION_OK);
1290   RunLoop();
1291   StopSyncScheduler();
1292 }
1293
1294 TEST_F(SyncSchedulerTest, SuccessfulRetry) {
1295   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1296
1297   SyncShareTimes times;
1298   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(10);
1299   scheduler()->OnReceivedGuRetryDelay(delay);
1300   EXPECT_EQ(delay, GetRetryTimerDelay());
1301
1302   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1303       .WillOnce(
1304           DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1305                 RecordSyncShare(&times)));
1306
1307   // Run to wait for retrying.
1308   RunLoop();
1309
1310   StopSyncScheduler();
1311 }
1312
1313 TEST_F(SyncSchedulerTest, FailedRetry) {
1314   UseMockDelayProvider();
1315   EXPECT_CALL(*delay(), GetDelay(_))
1316       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(10)));
1317
1318   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1319
1320   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(10);
1321   scheduler()->OnReceivedGuRetryDelay(delay);
1322
1323   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1324       .WillOnce(
1325           DoAll(Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
1326                 QuitLoopNowAction()));
1327
1328   // Run to wait for retrying.
1329   RunLoop();
1330
1331   EXPECT_TRUE(scheduler()->IsBackingOff());
1332   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1333       .WillOnce(
1334           DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1335                 QuitLoopNowAction()));
1336
1337   // Run to wait for second retrying.
1338   RunLoop();
1339
1340   StopSyncScheduler();
1341 }
1342
1343 ACTION_P2(VerifyRetryTimerDelay, scheduler_test, expected_delay) {
1344   EXPECT_EQ(expected_delay, scheduler_test->GetRetryTimerDelay());
1345 }
1346
1347 TEST_F(SyncSchedulerTest, ReceiveNewRetryDelay) {
1348   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1349
1350   SyncShareTimes times;
1351   base::TimeDelta delay1 = base::TimeDelta::FromMilliseconds(100);
1352   base::TimeDelta delay2 = base::TimeDelta::FromMilliseconds(200);
1353
1354   scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1355   scheduler()->OnReceivedGuRetryDelay(delay1);
1356   EXPECT_EQ(delay1, GetRetryTimerDelay());
1357
1358   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1359       .WillOnce(DoAll(
1360           WithoutArgs(VerifyRetryTimerDelay(this, delay1)),
1361           WithArg<2>(sessions::test_util::SimulateGuRetryDelayCommand(delay2)),
1362           RecordSyncShare(&times)));
1363
1364   // Run nudge GU.
1365   RunLoop();
1366   EXPECT_EQ(delay2, GetRetryTimerDelay());
1367
1368   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1369       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1370                       RecordSyncShare(&times)));
1371
1372   // Run to wait for retrying.
1373   RunLoop();
1374
1375   StopSyncScheduler();
1376 }
1377
1378 }  // namespace syncer