chrome/browser/metrics/thread_watcher_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <math.h>
   6
   7 #include "base/basictypes.h"
   8 #include "base/bind.h"
   9 #include "base/location.h"
  10 #include "base/logging.h"
  11 #include "base/memory/scoped_ptr.h"
  12 #include "base/message_loop/message_loop.h"
  13 #include "base/run_loop.h"
  14 #include "base/single_thread_task_runner.h"
  15 #include "base/strings/string_number_conversions.h"
  16 #include "base/strings/string_split.h"
  17 #include "base/strings/string_tokenizer.h"
  18 #include "base/synchronization/condition_variable.h"
  19 #include "base/synchronization/lock.h"
  20 #include "base/threading/platform_thread.h"
  21 #include "base/time/time.h"
  22 #include "build/build_config.h"
  23 #include "chrome/browser/metrics/thread_watcher.h"
  24 #include "chrome/common/chrome_switches.h"
  25 #include "content/public/test/test_browser_thread.h"
  26 #include "testing/gtest/include/gtest/gtest.h"
  27 #include "testing/platform_test.h"
  28
  29 using base::TimeDelta;
  30 using base::TimeTicks;
  31 using content::BrowserThread;
  32
  33 enum State {
  34   INITIALIZED,        // Created ThreadWatch object.
  35   ACTIVATED,          // Thread watching activated.
  36   SENT_PING,          // Sent ping message to watched thread.
  37   RECEIVED_PONG,      // Received Pong message.
  38   DEACTIVATED,        // Thread watching de-activated.
  39 };
  40
  41 enum WaitState {
  42   UNINITIALIZED,
  43   STARTED_WAITING,    // Start waiting for state_ to change to expected_state.
  44   STOPPED_WAITING,    // Done with the waiting.
  45   ALL_DONE,           // Done with waiting for STOPPED_WAITING.
  46 };
  47
  48 enum CheckResponseState {
  49   UNKNOWN,
  50   SUCCESSFUL,         // CheckResponse was successful.
  51   FAILED,             // CheckResponse has failed.
  52 };
  53
  54 // This class helps to track and manipulate thread state during tests. This
  55 // class also has utility method to simulate hanging of watched thread by making
  56 // the watched thread wait for a very long time by posting a task on watched
  57 // thread that keeps it busy. It also has an utility method to block running of
  58 // tests until ThreadWatcher object's post-condition state changes to an
  59 // expected state.
  60 class CustomThreadWatcher : public ThreadWatcher {
  61  public:
  62   base::Lock custom_lock_;
  63   base::ConditionVariable state_changed_;
  64   State thread_watcher_state_;
  65   WaitState wait_state_;
  66   CheckResponseState check_response_state_;
  67   uint64 ping_sent_;
  68   uint64 pong_received_;
  69   base::subtle::Atomic32 success_response_;
  70   base::subtle::Atomic32 failed_response_;
  71   base::TimeTicks saved_ping_time_;
  72   uint64 saved_ping_sequence_number_;
  73
  74   CustomThreadWatcher(const BrowserThread::ID thread_id,
  75                       const std::string thread_name,
  76                       const TimeDelta& sleep_time,
  77                       const TimeDelta& unresponsive_time)
  78       : ThreadWatcher(WatchingParams(thread_id, thread_name, sleep_time,
  79                       unresponsive_time, ThreadWatcherList::kUnresponsiveCount,
  80                       true, ThreadWatcherList::kLiveThreadsThreshold)),
  81         state_changed_(&custom_lock_),
  82         thread_watcher_state_(INITIALIZED),
  83         wait_state_(UNINITIALIZED),
  84         check_response_state_(UNKNOWN),
  85         ping_sent_(0),
  86         pong_received_(0),
  87         success_response_(0),
  88         failed_response_(0),
  89         saved_ping_time_(base::TimeTicks::Now()),
  90         saved_ping_sequence_number_(0) {
  91   }
  92
  93   State UpdateState(State new_state) {
  94     State old_state;
  95     {
  96       base::AutoLock auto_lock(custom_lock_);
  97       old_state = thread_watcher_state_;
  98       if (old_state != DEACTIVATED)
  99         thread_watcher_state_ = new_state;
 100       if (new_state == SENT_PING)
 101         ++ping_sent_;
 102       if (new_state == RECEIVED_PONG)
 103         ++pong_received_;
 104       saved_ping_time_ = ping_time();
 105       saved_ping_sequence_number_ = ping_sequence_number();
 106     }
 107     state_changed_.Broadcast();
 108     return old_state;
 109   }
 110
 111   WaitState UpdateWaitState(WaitState new_state) {
 112     WaitState old_state;
 113     {
 114       base::AutoLock auto_lock(custom_lock_);
 115       old_state = wait_state_;
 116       wait_state_ = new_state;
 117     }
 118     state_changed_.Broadcast();
 119     return old_state;
 120   }
 121
 122   void ActivateThreadWatching() override {
 123     State old_state = UpdateState(ACTIVATED);
 124     EXPECT_EQ(old_state, INITIALIZED);
 125     ThreadWatcher::ActivateThreadWatching();
 126   }
 127
 128   void DeActivateThreadWatching() override {
 129     State old_state = UpdateState(DEACTIVATED);
 130     EXPECT_TRUE(old_state == ACTIVATED || old_state == SENT_PING ||
 131                 old_state == RECEIVED_PONG);
 132     ThreadWatcher::DeActivateThreadWatching();
 133   }
 134
 135   void PostPingMessage() override {
 136     State old_state = UpdateState(SENT_PING);
 137     EXPECT_TRUE(old_state == ACTIVATED || old_state == RECEIVED_PONG);
 138     ThreadWatcher::PostPingMessage();
 139   }
 140
 141   void OnPongMessage(uint64 ping_sequence_number) override {
 142     State old_state = UpdateState(RECEIVED_PONG);
 143     EXPECT_TRUE(old_state == SENT_PING || old_state == DEACTIVATED);
 144     ThreadWatcher::OnPongMessage(ping_sequence_number);
 145   }
 146
 147   void OnCheckResponsiveness(uint64 ping_sequence_number) override {
 148     ThreadWatcher::OnCheckResponsiveness(ping_sequence_number);
 149     {
 150       base::AutoLock auto_lock(custom_lock_);
 151       if (responsive_) {
 152         base::subtle::Release_Store(&success_response_,
 153             base::subtle::Acquire_Load(&success_response_) + 1);
 154         check_response_state_ = SUCCESSFUL;
 155       } else {
 156         base::subtle::Release_Store(&failed_response_,
 157             base::subtle::Acquire_Load(&failed_response_) + 1);
 158         check_response_state_ = FAILED;
 159       }
 160     }
 161     // Broadcast to indicate we have checked responsiveness of the thread that
 162     // is watched.
 163     state_changed_.Broadcast();
 164   }
 165
 166   void WaitForWaitStateChange(TimeDelta wait_time, WaitState expected_state) {
 167     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 168     TimeTicks end_time = TimeTicks::Now() + wait_time;
 169     {
 170       base::AutoLock auto_lock(custom_lock_);
 171       while (wait_state_ != expected_state && TimeTicks::Now() < end_time)
 172         state_changed_.TimedWait(end_time - TimeTicks::Now());
 173     }
 174   }
 175
 176   void VeryLongMethod(TimeDelta wait_time) {
 177     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 178     WaitForWaitStateChange(wait_time, STOPPED_WAITING);
 179     UpdateWaitState(ALL_DONE);
 180   }
 181
 182   State WaitForStateChange(const TimeDelta& wait_time, State expected_state) {
 183     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 184     UpdateWaitState(STARTED_WAITING);
 185
 186     State exit_state = INITIALIZED;
 187     // Keep the thread that is running the tests waiting until ThreadWatcher
 188     // object's state changes to the expected_state or until wait_time elapses.
 189     for (uint32 i = 0; i < unresponsive_threshold_; ++i) {
 190         TimeTicks end_time = TimeTicks::Now() + wait_time;
 191         {
 192           base::AutoLock auto_lock(custom_lock_);
 193           while (thread_watcher_state_ != expected_state &&
 194                  TimeTicks::Now() < end_time) {
 195             TimeDelta state_change_wait_time = end_time - TimeTicks::Now();
 196             state_changed_.TimedWait(state_change_wait_time);
 197           }
 198           // Capture the thread_watcher_state_ before it changes and return it
 199           // to the caller.
 200           exit_state = thread_watcher_state_;
 201           if (exit_state == expected_state)
 202             break;
 203         }
 204     }
 205     UpdateWaitState(STOPPED_WAITING);
 206     return exit_state;
 207   }
 208
 209   CheckResponseState WaitForCheckResponse(const TimeDelta& wait_time,
 210                                           CheckResponseState expected_state) {
 211     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 212     UpdateWaitState(STARTED_WAITING);
 213
 214     CheckResponseState exit_state = UNKNOWN;
 215     // Keep the thread that is running the tests waiting until ThreadWatcher
 216     // object's check_response_state_ changes to the expected_state or until
 217     // wait_time elapses.
 218     for (uint32 i = 0; i < unresponsive_threshold_; ++i) {
 219         TimeTicks end_time = TimeTicks::Now() + wait_time;
 220         {
 221           base::AutoLock auto_lock(custom_lock_);
 222           while (check_response_state_ != expected_state &&
 223                  TimeTicks::Now() < end_time) {
 224             TimeDelta state_change_wait_time = end_time - TimeTicks::Now();
 225             state_changed_.TimedWait(state_change_wait_time);
 226           }
 227           // Capture the check_response_state_ before it changes and return it
 228           // to the caller.
 229           exit_state = check_response_state_;
 230           if (exit_state == expected_state)
 231             break;
 232         }
 233     }
 234     UpdateWaitState(STOPPED_WAITING);
 235     return exit_state;
 236   }
 237 };
 238
 239 class ThreadWatcherTest : public ::testing::Test {
 240  public:
 241   static const TimeDelta kSleepTime;
 242   static const TimeDelta kUnresponsiveTime;
 243   static const BrowserThread::ID io_thread_id;
 244   static const std::string io_thread_name;
 245   static const BrowserThread::ID db_thread_id;
 246   static const std::string db_thread_name;
 247   static const std::string crash_on_hang_seconds;
 248   static const std::string crash_on_hang_thread_names;
 249   static const std::string thread_names_and_live_threshold;
 250   static const std::string crash_on_hang_thread_data;
 251   CustomThreadWatcher* io_watcher_;
 252   CustomThreadWatcher* db_watcher_;
 253   ThreadWatcherList* thread_watcher_list_;
 254
 255   ThreadWatcherTest()
 256       : setup_complete_(&lock_),
 257         initialized_(false) {
 258     db_thread_.reset(new content::TestBrowserThread(BrowserThread::DB));
 259     io_thread_.reset(new content::TestBrowserThread(BrowserThread::IO));
 260     watchdog_thread_.reset(new WatchDogThread());
 261     db_thread_->StartAndWaitForTesting();
 262     io_thread_->StartAndWaitForTesting();
 263     watchdog_thread_->StartAndWaitForTesting();
 264
 265     WatchDogThread::PostTask(
 266         FROM_HERE,
 267         base::Bind(&ThreadWatcherTest::SetUpObjects, base::Unretained(this)));
 268
 269     WaitForSetUp(TimeDelta::FromMinutes(1));
 270   }
 271
 272   void SetUpObjects() {
 273     DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 274
 275     // Setup the registry for thread watchers.
 276     thread_watcher_list_ = new ThreadWatcherList();
 277
 278     // Create thread watcher object for the IO thread.
 279     io_watcher_ = new CustomThreadWatcher(io_thread_id, io_thread_name,
 280                                           kSleepTime, kUnresponsiveTime);
 281     EXPECT_EQ(io_watcher_, thread_watcher_list_->Find(io_thread_id));
 282
 283     // Create thread watcher object for the DB thread.
 284     db_watcher_ = new CustomThreadWatcher(
 285         db_thread_id, db_thread_name, kSleepTime, kUnresponsiveTime);
 286     EXPECT_EQ(db_watcher_, thread_watcher_list_->Find(db_thread_id));
 287
 288     {
 289       base::AutoLock lock(lock_);
 290       initialized_ = true;
 291     }
 292     setup_complete_.Signal();
 293   }
 294
 295   void WaitForSetUp(TimeDelta wait_time) {
 296     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 297     TimeTicks end_time = TimeTicks::Now() + wait_time;
 298     {
 299       base::AutoLock auto_lock(lock_);
 300       while (!initialized_ && TimeTicks::Now() < end_time)
 301         setup_complete_.TimedWait(end_time - TimeTicks::Now());
 302     }
 303   }
 304
 305   ~ThreadWatcherTest() override {
 306     ThreadWatcherList::DeleteAll();
 307     io_watcher_ = NULL;
 308     db_watcher_ = NULL;
 309     io_thread_.reset();
 310     db_thread_.reset();
 311     watchdog_thread_.reset();
 312     thread_watcher_list_ = NULL;
 313   }
 314
 315  private:
 316   base::MessageLoop message_loop_;
 317   base::Lock lock_;
 318   base::ConditionVariable setup_complete_;
 319   bool initialized_;
 320   scoped_ptr<content::TestBrowserThread> db_thread_;
 321   scoped_ptr<content::TestBrowserThread> io_thread_;
 322   scoped_ptr<WatchDogThread> watchdog_thread_;
 323 };
 324
 325 // Define static constants.
 326 const TimeDelta ThreadWatcherTest::kSleepTime =
 327     TimeDelta::FromMilliseconds(50);
 328 const TimeDelta ThreadWatcherTest::kUnresponsiveTime =
 329     TimeDelta::FromMilliseconds(500);
 330 const BrowserThread::ID ThreadWatcherTest::io_thread_id = BrowserThread::IO;
 331 const std::string ThreadWatcherTest::io_thread_name = "IO";
 332 const BrowserThread::ID ThreadWatcherTest::db_thread_id = BrowserThread::DB;
 333 const std::string ThreadWatcherTest::db_thread_name = "DB";
 334 const std::string ThreadWatcherTest::crash_on_hang_thread_names = "UI,IO";
 335 const std::string ThreadWatcherTest::thread_names_and_live_threshold =
 336     "UI:4,IO:4";
 337 const std::string ThreadWatcherTest::crash_on_hang_thread_data =
 338     "UI:5:12,IO:5:12,FILE:5:12";
 339
 340 TEST_F(ThreadWatcherTest, ThreadNamesOnlyArgs) {
 341   // Setup command_line arguments.
 342   base::CommandLine command_line(base::CommandLine::NO_PROGRAM);
 343   command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
 344                                  crash_on_hang_thread_names);
 345
 346   // Parse command_line arguments.
 347   ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
 348   uint32 unresponsive_threshold;
 349   ThreadWatcherList::ParseCommandLine(command_line,
 350                                       &unresponsive_threshold,
 351                                       &crash_on_hang_threads);
 352
 353   // Verify the data.
 354   base::StringTokenizer tokens(crash_on_hang_thread_names, ",");
 355   while (tokens.GetNext()) {
 356     std::vector<base::StringPiece> values = base::SplitStringPiece(
 357         tokens.token_piece(), ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
 358     std::string thread_name = values[0].as_string();
 359
 360     ThreadWatcherList::CrashOnHangThreadMap::iterator it =
 361         crash_on_hang_threads.find(thread_name);
 362     bool crash_on_hang = (it != crash_on_hang_threads.end());
 363     EXPECT_TRUE(crash_on_hang);
 364     EXPECT_LT(0u, it->second.live_threads_threshold);
 365     EXPECT_LT(0u, it->second.unresponsive_threshold);
 366   }
 367 }
 368
 369 TEST_F(ThreadWatcherTest, ThreadNamesAndLiveThresholdArgs) {
 370   // Setup command_line arguments.
 371   base::CommandLine command_line(base::CommandLine::NO_PROGRAM);
 372   command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
 373                                  thread_names_and_live_threshold);
 374
 375   // Parse command_line arguments.
 376   ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
 377   uint32 unresponsive_threshold;
 378   ThreadWatcherList::ParseCommandLine(command_line,
 379                                       &unresponsive_threshold,
 380                                       &crash_on_hang_threads);
 381
 382   // Verify the data.
 383   base::StringTokenizer tokens(thread_names_and_live_threshold, ",");
 384   while (tokens.GetNext()) {
 385     std::vector<base::StringPiece> values = base::SplitStringPiece(
 386         tokens.token_piece(), ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
 387     std::string thread_name = values[0].as_string();
 388
 389     ThreadWatcherList::CrashOnHangThreadMap::iterator it =
 390         crash_on_hang_threads.find(thread_name);
 391     bool crash_on_hang = (it != crash_on_hang_threads.end());
 392     EXPECT_TRUE(crash_on_hang);
 393     EXPECT_EQ(4u, it->second.live_threads_threshold);
 394     EXPECT_LT(0u, it->second.unresponsive_threshold);
 395   }
 396 }
 397
 398 TEST_F(ThreadWatcherTest, CrashOnHangThreadsAllArgs) {
 399   // Setup command_line arguments.
 400   base::CommandLine command_line(base::CommandLine::NO_PROGRAM);
 401   command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
 402                                  crash_on_hang_thread_data);
 403
 404   // Parse command_line arguments.
 405   ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
 406   uint32 unresponsive_threshold;
 407   ThreadWatcherList::ParseCommandLine(command_line,
 408                                       &unresponsive_threshold,
 409                                       &crash_on_hang_threads);
 410
 411   // Verify the data.
 412   base::StringTokenizer tokens(crash_on_hang_thread_data, ",");
 413   while (tokens.GetNext()) {
 414     std::vector<base::StringPiece> values = base::SplitStringPiece(
 415         tokens.token_piece(), ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
 416     std::string thread_name = values[0].as_string();
 417
 418     ThreadWatcherList::CrashOnHangThreadMap::iterator it =
 419         crash_on_hang_threads.find(thread_name);
 420
 421     bool crash_on_hang = (it != crash_on_hang_threads.end());
 422     EXPECT_TRUE(crash_on_hang);
 423
 424     uint32 crash_live_threads_threshold = it->second.live_threads_threshold;
 425     EXPECT_EQ(5u, crash_live_threads_threshold);
 426
 427     uint32 crash_unresponsive_threshold = it->second.unresponsive_threshold;
 428     uint32 crash_on_unresponsive_seconds =
 429         ThreadWatcherList::kUnresponsiveSeconds * crash_unresponsive_threshold;
 430     EXPECT_EQ(12u, crash_on_unresponsive_seconds);
 431   }
 432 }
 433
 434 // Test registration. When thread_watcher_list_ goes out of scope after
 435 // TearDown, all thread watcher objects will be deleted.
 436 TEST_F(ThreadWatcherTest, Registration) {
 437   // Check ThreadWatcher object has all correct parameters.
 438   EXPECT_EQ(io_thread_id, io_watcher_->thread_id());
 439   EXPECT_EQ(io_thread_name, io_watcher_->thread_name());
 440   EXPECT_EQ(kSleepTime, io_watcher_->sleep_time());
 441   EXPECT_EQ(kUnresponsiveTime, io_watcher_->unresponsive_time());
 442   EXPECT_FALSE(io_watcher_->active());
 443
 444   // Check ThreadWatcher object of watched DB thread has correct data.
 445   EXPECT_EQ(db_thread_id, db_watcher_->thread_id());
 446   EXPECT_EQ(db_thread_name, db_watcher_->thread_name());
 447   EXPECT_EQ(kSleepTime, db_watcher_->sleep_time());
 448   EXPECT_EQ(kUnresponsiveTime, db_watcher_->unresponsive_time());
 449   EXPECT_FALSE(db_watcher_->active());
 450 }
 451
 452 // Test ActivateThreadWatching and DeActivateThreadWatching of IO thread. This
 453 // method also checks that pong message was sent by the watched thread and pong
 454 // message was received by the WatchDogThread. It also checks that
 455 // OnCheckResponsiveness has verified the ping-pong mechanism and the watched
 456 // thread is not hung.
 457 TEST_F(ThreadWatcherTest, ThreadResponding) {
 458   TimeTicks time_before_ping = TimeTicks::Now();
 459   // Activate watching IO thread.
 460   WatchDogThread::PostTask(
 461       FROM_HERE,
 462       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 463                  base::Unretained(io_watcher_)));
 464
 465   // Activate would have started ping/pong messaging. Expect atleast one
 466   // ping/pong messaging sequence to happen.
 467   io_watcher_->WaitForStateChange(kSleepTime + TimeDelta::FromMinutes(1),
 468                                   RECEIVED_PONG);
 469   EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0));
 470   EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0));
 471   EXPECT_TRUE(io_watcher_->active());
 472   EXPECT_GE(io_watcher_->saved_ping_time_, time_before_ping);
 473   EXPECT_GE(io_watcher_->saved_ping_sequence_number_, static_cast<uint64>(0));
 474
 475   // Verify watched thread is responding with ping/pong messaging.
 476   io_watcher_->WaitForCheckResponse(
 477       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
 478   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
 479       static_cast<base::subtle::Atomic32>(0));
 480   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
 481       static_cast<base::subtle::Atomic32>(0));
 482
 483   // DeActivate thread watching for shutdown.
 484   WatchDogThread::PostTask(
 485       FROM_HERE,
 486       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 487       base::Unretained(io_watcher_)));
 488 }
 489
 490 // This test posts a task on watched thread that takes very long time (this is
 491 // to simulate hanging of watched thread). It then checks for
 492 // OnCheckResponsiveness raising an alert (OnCheckResponsiveness returns false
 493 // if the watched thread is not responding).
 494 TEST_F(ThreadWatcherTest, ThreadNotResponding) {
 495   // Simulate hanging of watched thread by making the watched thread wait for a
 496   // very long time by posting a task on watched thread that keeps it busy.
 497   // It is safe to use base::Unretained because test is waiting for the method
 498   // to finish.
 499   BrowserThread::PostTask(
 500       io_thread_id,
 501       FROM_HERE,
 502       base::Bind(&CustomThreadWatcher::VeryLongMethod,
 503                  base::Unretained(io_watcher_),
 504                  kUnresponsiveTime * 10));
 505
 506   // Activate thread watching.
 507   WatchDogThread::PostTask(
 508       FROM_HERE,
 509       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 510                  base::Unretained(io_watcher_)));
 511
 512   // Verify watched thread is not responding for ping messages.
 513   io_watcher_->WaitForCheckResponse(
 514       kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED);
 515   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
 516       static_cast<base::subtle::Atomic32>(0));
 517   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
 518       static_cast<base::subtle::Atomic32>(0));
 519
 520   // DeActivate thread watching for shutdown.
 521   WatchDogThread::PostTask(
 522       FROM_HERE,
 523       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 524                  base::Unretained(io_watcher_)));
 525
 526   // Wait for the io_watcher_'s VeryLongMethod to finish.
 527   io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE);
 528 }
 529
 530 // Test watching of multiple threads with all threads not responding.
 531 TEST_F(ThreadWatcherTest, MultipleThreadsResponding) {
 532   // Check for DB thread to perform ping/pong messaging.
 533   WatchDogThread::PostTask(
 534       FROM_HERE,
 535       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 536                  base::Unretained(db_watcher_)));
 537
 538   // Check for IO thread to perform ping/pong messaging.
 539   WatchDogThread::PostTask(
 540       FROM_HERE,
 541       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 542                  base::Unretained(io_watcher_)));
 543
 544   // Verify DB thread is responding with ping/pong messaging.
 545   db_watcher_->WaitForCheckResponse(
 546       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
 547   EXPECT_GT(db_watcher_->ping_sent_, static_cast<uint64>(0));
 548   EXPECT_GT(db_watcher_->pong_received_, static_cast<uint64>(0));
 549   EXPECT_GE(db_watcher_->ping_sequence_number_, static_cast<uint64>(0));
 550   EXPECT_GT(base::subtle::NoBarrier_Load(&(db_watcher_->success_response_)),
 551       static_cast<base::subtle::Atomic32>(0));
 552   EXPECT_EQ(base::subtle::NoBarrier_Load(&(db_watcher_->failed_response_)),
 553       static_cast<base::subtle::Atomic32>(0));
 554
 555   // Verify IO thread is responding with ping/pong messaging.
 556   io_watcher_->WaitForCheckResponse(
 557       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
 558   EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0));
 559   EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0));
 560   EXPECT_GE(io_watcher_->ping_sequence_number_, static_cast<uint64>(0));
 561   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
 562       static_cast<base::subtle::Atomic32>(0));
 563   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
 564       static_cast<base::subtle::Atomic32>(0));
 565
 566   // DeActivate thread watching for shutdown.
 567   WatchDogThread::PostTask(
 568       FROM_HERE,
 569       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 570                  base::Unretained(io_watcher_)));
 571
 572   WatchDogThread::PostTask(
 573       FROM_HERE,
 574       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 575                  base::Unretained(db_watcher_)));
 576 }
 577
 578 // Test watching of multiple threads with one of the threads not responding.
 579 TEST_F(ThreadWatcherTest, MultipleThreadsNotResponding) {
 580   // Simulate hanging of watched thread by making the watched thread wait for a
 581   // very long time by posting a task on watched thread that keeps it busy.
 582   // It is safe ot use base::Unretained because test is waiting for the method
 583   // to finish.
 584   BrowserThread::PostTask(
 585       io_thread_id,
 586       FROM_HERE,
 587       base::Bind(&CustomThreadWatcher::VeryLongMethod,
 588                  base::Unretained(io_watcher_),
 589                  kUnresponsiveTime * 10));
 590
 591   // Activate watching of DB thread.
 592   WatchDogThread::PostTask(
 593       FROM_HERE,
 594       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 595                  base::Unretained(db_watcher_)));
 596
 597   // Activate watching of IO thread.
 598   WatchDogThread::PostTask(
 599       FROM_HERE,
 600       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 601                  base::Unretained(io_watcher_)));
 602
 603   // Verify DB thread is responding with ping/pong messaging.
 604   db_watcher_->WaitForCheckResponse(
 605       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
 606   EXPECT_GT(base::subtle::NoBarrier_Load(&(db_watcher_->success_response_)),
 607       static_cast<base::subtle::Atomic32>(0));
 608   EXPECT_EQ(base::subtle::NoBarrier_Load(&(db_watcher_->failed_response_)),
 609       static_cast<base::subtle::Atomic32>(0));
 610
 611   // Verify IO thread is not responding for ping messages.
 612   io_watcher_->WaitForCheckResponse(
 613       kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED);
 614   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
 615       static_cast<base::subtle::Atomic32>(0));
 616   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
 617       static_cast<base::subtle::Atomic32>(0));
 618
 619   // DeActivate thread watching for shutdown.
 620   WatchDogThread::PostTask(
 621       FROM_HERE,
 622       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 623                  base::Unretained(io_watcher_)));
 624   WatchDogThread::PostTask(
 625       FROM_HERE,
 626       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 627                  base::Unretained(db_watcher_)));
 628
 629   // Wait for the io_watcher_'s VeryLongMethod to finish.
 630   io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE);
 631 }
 632
 633 class ThreadWatcherListTest : public ::testing::Test {
 634  protected:
 635   ThreadWatcherListTest()
 636       : done_(&lock_),
 637         state_available_(false),
 638         has_thread_watcher_list_(false),
 639         stopped_(false) {
 640   }
 641
 642   void ReadStateOnWatchDogThread() {
 643     CHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 644     {
 645       base::AutoLock auto_lock(lock_);
 646       has_thread_watcher_list_ =
 647           ThreadWatcherList::g_thread_watcher_list_ != NULL;
 648       stopped_ = ThreadWatcherList::g_stopped_;
 649       state_available_ = true;
 650     }
 651     done_.Signal();
 652   }
 653
 654   void CheckState(bool has_thread_watcher_list,
 655                   bool stopped,
 656                   const char* const msg) {
 657     CHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 658     {
 659       base::AutoLock auto_lock(lock_);
 660       state_available_ = false;
 661     }
 662
 663     WatchDogThread::PostTask(
 664         FROM_HERE,
 665         base::Bind(&ThreadWatcherListTest::ReadStateOnWatchDogThread,
 666                    base::Unretained(this)));
 667     {
 668       base::AutoLock auto_lock(lock_);
 669       while (!state_available_)
 670         done_.Wait();
 671
 672       EXPECT_EQ(has_thread_watcher_list, has_thread_watcher_list_) << msg;
 673       EXPECT_EQ(stopped, stopped_) << msg;
 674     }
 675   }
 676
 677   base::Lock lock_;
 678   base::ConditionVariable done_;
 679
 680   bool state_available_;
 681   bool has_thread_watcher_list_;
 682   bool stopped_;
 683 };
 684
 685 TEST_F(ThreadWatcherListTest, Restart) {
 686   ThreadWatcherList::g_initialize_delay_seconds = 1;
 687
 688   base::MessageLoopForUI message_loop_for_ui;
 689   content::TestBrowserThread ui_thread(BrowserThread::UI, &message_loop_for_ui);
 690
 691   scoped_ptr<WatchDogThread> watchdog_thread_(new WatchDogThread());
 692   watchdog_thread_->StartAndWaitForTesting();
 693
 694   // See http://crbug.com/347887.
 695   // StartWatchingAll() will PostDelayedTask to create g_thread_watcher_list_,
 696   // whilst StopWatchingAll() will just PostTask to destroy it.
 697   // Ensure that when Stop is called, Start will NOT create
 698   // g_thread_watcher_list_ later on.
 699   ThreadWatcherList::StartWatchingAll(*base::CommandLine::ForCurrentProcess());
 700   ThreadWatcherList::StopWatchingAll();
 701   message_loop_for_ui.task_runner()->PostDelayedTask(
 702       FROM_HERE, message_loop_for_ui.QuitClosure(),
 703       base::TimeDelta::FromSeconds(
 704           ThreadWatcherList::g_initialize_delay_seconds));
 705   message_loop_for_ui.Run();
 706
 707   CheckState(false /* has_thread_watcher_list */,
 708              true /* stopped */,
 709              "Start / Stopped");
 710
 711   // Proceed with just |StartWatchingAll| and ensure it'll be started.
 712   ThreadWatcherList::StartWatchingAll(*base::CommandLine::ForCurrentProcess());
 713   message_loop_for_ui.task_runner()->PostDelayedTask(
 714       FROM_HERE, message_loop_for_ui.QuitClosure(),
 715       base::TimeDelta::FromSeconds(
 716           ThreadWatcherList::g_initialize_delay_seconds + 1));
 717   message_loop_for_ui.Run();
 718
 719   CheckState(true /* has_thread_watcher_list */,
 720              false /* stopped */,
 721              "Started");
 722
 723   // Finally, StopWatchingAll() must stop.
 724   ThreadWatcherList::StopWatchingAll();
 725   message_loop_for_ui.task_runner()->PostDelayedTask(
 726       FROM_HERE, message_loop_for_ui.QuitClosure(),
 727       base::TimeDelta::FromSeconds(
 728           ThreadWatcherList::g_initialize_delay_seconds));
 729   message_loop_for_ui.Run();
 730
 731   CheckState(false /* has_thread_watcher_list */,
 732              true /* stopped */,
 733              "Stopped");
 734 }