chrome/browser/metrics/thread_watcher.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/metrics/thread_watcher.h"
   6
   7 #include <math.h>  // ceil
   8
   9 #include "base/bind.h"
  10 #include "base/compiler_specific.h"
  11 #include "base/debug/dump_without_crashing.h"
  12 #include "base/lazy_instance.h"
  13 #include "base/strings/string_number_conversions.h"
  14 #include "base/strings/string_split.h"
  15 #include "base/strings/string_tokenizer.h"
  16 #include "base/strings/stringprintf.h"
  17 #include "base/threading/thread_restrictions.h"
  18 #include "build/build_config.h"
  19 #include "chrome/browser/chrome_notification_types.h"
  20 #include "chrome/browser/metrics/thread_watcher_report_hang.h"
  21 #include "chrome/common/chrome_switches.h"
  22 #include "chrome/common/chrome_version_info.h"
  23 #include "chrome/common/logging_chrome.h"
  24 #include "content/public/browser/notification_service.h"
  25
  26 #if defined(OS_WIN)
  27 #include "base/win/windows_version.h"
  28 #endif
  29
  30 using content::BrowserThread;
  31
  32 // ThreadWatcher methods and members.
  33 ThreadWatcher::ThreadWatcher(const WatchingParams& params)
  34     : thread_id_(params.thread_id),
  35       thread_name_(params.thread_name),
  36       watched_loop_(
  37           BrowserThread::GetMessageLoopProxyForThread(params.thread_id)),
  38       sleep_time_(params.sleep_time),
  39       unresponsive_time_(params.unresponsive_time),
  40       ping_time_(base::TimeTicks::Now()),
  41       pong_time_(ping_time_),
  42       ping_sequence_number_(0),
  43       active_(false),
  44       ping_count_(params.unresponsive_threshold),
  45       response_time_histogram_(NULL),
  46       unresponsive_time_histogram_(NULL),
  47       unresponsive_count_(0),
  48       hung_processing_complete_(false),
  49       unresponsive_threshold_(params.unresponsive_threshold),
  50       crash_on_hang_(params.crash_on_hang),
  51       live_threads_threshold_(params.live_threads_threshold),
  52       weak_ptr_factory_(this) {
  53   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  54   Initialize();
  55 }
  56
  57 ThreadWatcher::~ThreadWatcher() {}
  58
  59 // static
  60 void ThreadWatcher::StartWatching(const WatchingParams& params) {
  61   DCHECK_GE(params.sleep_time.InMilliseconds(), 0);
  62   DCHECK_GE(params.unresponsive_time.InMilliseconds(),
  63             params.sleep_time.InMilliseconds());
  64
  65   // If we are not on WatchDogThread, then post a task to call StartWatching on
  66   // WatchDogThread.
  67   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
  68     WatchDogThread::PostTask(
  69         FROM_HERE,
  70         base::Bind(&ThreadWatcher::StartWatching, params));
  71     return;
  72   }
  73
  74   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  75
  76   // Create a new thread watcher object for the given thread and activate it.
  77   ThreadWatcher* watcher = new ThreadWatcher(params);
  78
  79   DCHECK(watcher);
  80   // If we couldn't register the thread watcher object, we are shutting down,
  81   // then don't activate thread watching.
  82   if (!ThreadWatcherList::IsRegistered(params.thread_id))
  83     return;
  84   watcher->ActivateThreadWatching();
  85 }
  86
  87 void ThreadWatcher::ActivateThreadWatching() {
  88   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  89   if (active_) return;
  90   active_ = true;
  91   ping_count_ = unresponsive_threshold_;
  92   ResetHangCounters();
  93   base::MessageLoop::current()->PostTask(
  94       FROM_HERE,
  95       base::Bind(&ThreadWatcher::PostPingMessage,
  96                  weak_ptr_factory_.GetWeakPtr()));
  97 }
  98
  99 void ThreadWatcher::DeActivateThreadWatching() {
 100   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 101   active_ = false;
 102   ping_count_ = 0;
 103   weak_ptr_factory_.InvalidateWeakPtrs();
 104 }
 105
 106 void ThreadWatcher::WakeUp() {
 107   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 108   // There is some user activity, PostPingMessage task of thread watcher if
 109   // needed.
 110   if (!active_) return;
 111
 112   // Throw away the previous |unresponsive_count_| and start over again. Just
 113   // before going to sleep, |unresponsive_count_| could be very close to
 114   // |unresponsive_threshold_| and when user becomes active,
 115   // |unresponsive_count_| can go over |unresponsive_threshold_| if there was no
 116   // response for ping messages. Reset |unresponsive_count_| to start measuring
 117   // the unresponsiveness of the threads when system becomes active.
 118   unresponsive_count_ = 0;
 119
 120   if (ping_count_ <= 0) {
 121     ping_count_ = unresponsive_threshold_;
 122     ResetHangCounters();
 123     PostPingMessage();
 124   } else {
 125     ping_count_ = unresponsive_threshold_;
 126   }
 127 }
 128
 129 void ThreadWatcher::PostPingMessage() {
 130   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 131   // If we have stopped watching or if the user is idle, then stop sending
 132   // ping messages.
 133   if (!active_ || ping_count_ <= 0)
 134     return;
 135
 136   // Save the current time when we have sent ping message.
 137   ping_time_ = base::TimeTicks::Now();
 138
 139   // Send a ping message to the watched thread. Callback will be called on
 140   // the WatchDogThread.
 141   base::Closure callback(
 142       base::Bind(&ThreadWatcher::OnPongMessage, weak_ptr_factory_.GetWeakPtr(),
 143                  ping_sequence_number_));
 144   if (watched_loop_->PostTask(
 145           FROM_HERE,
 146           base::Bind(&ThreadWatcher::OnPingMessage, thread_id_,
 147                      callback))) {
 148       // Post a task to check the responsiveness of watched thread.
 149       base::MessageLoop::current()->PostDelayedTask(
 150           FROM_HERE,
 151           base::Bind(&ThreadWatcher::OnCheckResponsiveness,
 152                      weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
 153           unresponsive_time_);
 154   } else {
 155     // Watched thread might have gone away, stop watching it.
 156     DeActivateThreadWatching();
 157   }
 158 }
 159
 160 void ThreadWatcher::OnPongMessage(uint64 ping_sequence_number) {
 161   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 162
 163   // Record watched thread's response time.
 164   base::TimeTicks now = base::TimeTicks::Now();
 165   base::TimeDelta response_time = now - ping_time_;
 166   response_time_histogram_->AddTime(response_time);
 167
 168   // Save the current time when we have got pong message.
 169   pong_time_ = now;
 170
 171   // Check if there are any extra pings in flight.
 172   DCHECK_EQ(ping_sequence_number_, ping_sequence_number);
 173   if (ping_sequence_number_ != ping_sequence_number)
 174     return;
 175
 176   // Increment sequence number for the next ping message to indicate watched
 177   // thread is responsive.
 178   ++ping_sequence_number_;
 179
 180   // If we have stopped watching or if the user is idle, then stop sending
 181   // ping messages.
 182   if (!active_ || --ping_count_ <= 0)
 183     return;
 184
 185   base::MessageLoop::current()->PostDelayedTask(
 186       FROM_HERE,
 187       base::Bind(&ThreadWatcher::PostPingMessage,
 188                  weak_ptr_factory_.GetWeakPtr()),
 189       sleep_time_);
 190 }
 191
 192 void ThreadWatcher::OnCheckResponsiveness(uint64 ping_sequence_number) {
 193   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 194   // If we have stopped watching then consider thread as responding.
 195   if (!active_) {
 196     responsive_ = true;
 197     return;
 198   }
 199   // If the latest ping_sequence_number_ is not same as the ping_sequence_number
 200   // that is passed in, then we can assume OnPongMessage was called.
 201   // OnPongMessage increments ping_sequence_number_.
 202   if (ping_sequence_number_ != ping_sequence_number) {
 203     // Reset unresponsive_count_ to zero because we got a response from the
 204     // watched thread.
 205     ResetHangCounters();
 206
 207     responsive_ = true;
 208     return;
 209   }
 210   // Record that we got no response from watched thread.
 211   GotNoResponse();
 212
 213   // Post a task to check the responsiveness of watched thread.
 214   base::MessageLoop::current()->PostDelayedTask(
 215       FROM_HERE,
 216       base::Bind(&ThreadWatcher::OnCheckResponsiveness,
 217                  weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
 218       unresponsive_time_);
 219   responsive_ = false;
 220 }
 221
 222 void ThreadWatcher::Initialize() {
 223   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 224   ThreadWatcherList::Register(this);
 225
 226   const std::string response_time_histogram_name =
 227       "ThreadWatcher.ResponseTime." + thread_name_;
 228   response_time_histogram_ = base::Histogram::FactoryTimeGet(
 229       response_time_histogram_name,
 230       base::TimeDelta::FromMilliseconds(1),
 231       base::TimeDelta::FromSeconds(100), 50,
 232       base::Histogram::kUmaTargetedHistogramFlag);
 233
 234   const std::string unresponsive_time_histogram_name =
 235       "ThreadWatcher.Unresponsive." + thread_name_;
 236   unresponsive_time_histogram_ = base::Histogram::FactoryTimeGet(
 237       unresponsive_time_histogram_name,
 238       base::TimeDelta::FromMilliseconds(1),
 239       base::TimeDelta::FromSeconds(100), 50,
 240       base::Histogram::kUmaTargetedHistogramFlag);
 241
 242   const std::string responsive_count_histogram_name =
 243       "ThreadWatcher.ResponsiveThreads." + thread_name_;
 244   responsive_count_histogram_ = base::LinearHistogram::FactoryGet(
 245       responsive_count_histogram_name, 1, 10, 11,
 246       base::Histogram::kUmaTargetedHistogramFlag);
 247
 248   const std::string unresponsive_count_histogram_name =
 249       "ThreadWatcher.UnresponsiveThreads." + thread_name_;
 250   unresponsive_count_histogram_ = base::LinearHistogram::FactoryGet(
 251       unresponsive_count_histogram_name, 1, 10, 11,
 252       base::Histogram::kUmaTargetedHistogramFlag);
 253 }
 254
 255 // static
 256 void ThreadWatcher::OnPingMessage(const BrowserThread::ID& thread_id,
 257                                   const base::Closure& callback_task) {
 258   // This method is called on watched thread.
 259   DCHECK(BrowserThread::CurrentlyOn(thread_id));
 260   WatchDogThread::PostTask(FROM_HERE, callback_task);
 261 }
 262
 263 void ThreadWatcher::ResetHangCounters() {
 264   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 265   unresponsive_count_ = 0;
 266   hung_processing_complete_ = false;
 267 }
 268
 269 void ThreadWatcher::GotNoResponse() {
 270   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 271
 272   ++unresponsive_count_;
 273   if (!IsVeryUnresponsive())
 274     return;
 275
 276   // Record total unresponsive_time since last pong message.
 277   base::TimeDelta unresponse_time = base::TimeTicks::Now() - pong_time_;
 278   unresponsive_time_histogram_->AddTime(unresponse_time);
 279
 280   // We have already collected stats for the non-responding watched thread.
 281   if (hung_processing_complete_)
 282     return;
 283
 284   // Record how other threads are responding.
 285   uint32 responding_thread_count = 0;
 286   uint32 unresponding_thread_count = 0;
 287   ThreadWatcherList::GetStatusOfThreads(&responding_thread_count,
 288                                         &unresponding_thread_count);
 289
 290   // Record how many watched threads are responding.
 291   responsive_count_histogram_->Add(responding_thread_count);
 292
 293   // Record how many watched threads are not responding.
 294   unresponsive_count_histogram_->Add(unresponding_thread_count);
 295
 296   // Crash the browser if the watched thread is to be crashed on hang and if the
 297   // number of other threads responding is less than or equal to
 298   // live_threads_threshold_ and at least one other thread is responding.
 299   if (crash_on_hang_ &&
 300       responding_thread_count > 0 &&
 301       responding_thread_count <= live_threads_threshold_) {
 302     static bool crashed_once = false;
 303     if (!crashed_once) {
 304       crashed_once = true;
 305       metrics::CrashBecauseThreadWasUnresponsive(thread_id_);
 306     }
 307   }
 308
 309   hung_processing_complete_ = true;
 310 }
 311
 312 bool ThreadWatcher::IsVeryUnresponsive() {
 313   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 314   return unresponsive_count_ >= unresponsive_threshold_;
 315 }
 316
 317 // ThreadWatcherList methods and members.
 318 //
 319 // static
 320 ThreadWatcherList* ThreadWatcherList::g_thread_watcher_list_ = NULL;
 321 // static
 322 bool ThreadWatcherList::g_stopped_ = false;
 323 // static
 324 const int ThreadWatcherList::kSleepSeconds = 1;
 325 // static
 326 const int ThreadWatcherList::kUnresponsiveSeconds = 2;
 327 // static
 328 const int ThreadWatcherList::kUnresponsiveCount = 9;
 329 // static
 330 const int ThreadWatcherList::kLiveThreadsThreshold = 2;
 331 // static, non-const for tests.
 332 int ThreadWatcherList::g_initialize_delay_seconds = 120;
 333
 334 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds(
 335     uint32 live_threads_threshold,
 336     uint32 unresponsive_threshold)
 337     : live_threads_threshold(live_threads_threshold),
 338       unresponsive_threshold(unresponsive_threshold) {
 339 }
 340
 341 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds()
 342     : live_threads_threshold(kLiveThreadsThreshold),
 343       unresponsive_threshold(kUnresponsiveCount) {
 344 }
 345
 346 // static
 347 void ThreadWatcherList::StartWatchingAll(
 348     const base::CommandLine& command_line) {
 349   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 350   uint32 unresponsive_threshold;
 351   CrashOnHangThreadMap crash_on_hang_threads;
 352   ParseCommandLine(command_line,
 353                    &unresponsive_threshold,
 354                    &crash_on_hang_threads);
 355
 356   ThreadWatcherObserver::SetupNotifications(
 357       base::TimeDelta::FromSeconds(kSleepSeconds * unresponsive_threshold));
 358
 359   WatchDogThread::PostTask(
 360       FROM_HERE,
 361       base::Bind(&ThreadWatcherList::SetStopped, false));
 362
 363   if (!WatchDogThread::PostDelayedTask(
 364           FROM_HERE,
 365           base::Bind(&ThreadWatcherList::InitializeAndStartWatching,
 366                      unresponsive_threshold,
 367                      crash_on_hang_threads),
 368           base::TimeDelta::FromSeconds(g_initialize_delay_seconds))) {
 369     // Disarm() the startup timebomb, if we couldn't post the task to start the
 370     // ThreadWatcher (becasue WatchDog thread is not running).
 371     StartupTimeBomb::DisarmStartupTimeBomb();
 372   }
 373 }
 374
 375 // static
 376 void ThreadWatcherList::StopWatchingAll() {
 377   // TODO(rtenneti): Enable ThreadWatcher.
 378   ThreadWatcherObserver::RemoveNotifications();
 379   DeleteAll();
 380 }
 381
 382 // static
 383 void ThreadWatcherList::Register(ThreadWatcher* watcher) {
 384   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 385   if (!g_thread_watcher_list_)
 386     return;
 387   DCHECK(!g_thread_watcher_list_->Find(watcher->thread_id()));
 388   g_thread_watcher_list_->registered_[watcher->thread_id()] = watcher;
 389 }
 390
 391 // static
 392 bool ThreadWatcherList::IsRegistered(const BrowserThread::ID thread_id) {
 393   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 394   return NULL != ThreadWatcherList::Find(thread_id);
 395 }
 396
 397 // static
 398 void ThreadWatcherList::GetStatusOfThreads(uint32* responding_thread_count,
 399                                            uint32* unresponding_thread_count) {
 400   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 401   *responding_thread_count = 0;
 402   *unresponding_thread_count = 0;
 403   if (!g_thread_watcher_list_)
 404     return;
 405
 406   for (RegistrationList::iterator it =
 407            g_thread_watcher_list_->registered_.begin();
 408        g_thread_watcher_list_->registered_.end() != it;
 409        ++it) {
 410     if (it->second->IsVeryUnresponsive())
 411       ++(*unresponding_thread_count);
 412     else
 413       ++(*responding_thread_count);
 414   }
 415 }
 416
 417 // static
 418 void ThreadWatcherList::WakeUpAll() {
 419   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 420   if (!g_thread_watcher_list_)
 421     return;
 422
 423   for (RegistrationList::iterator it =
 424            g_thread_watcher_list_->registered_.begin();
 425        g_thread_watcher_list_->registered_.end() != it;
 426        ++it)
 427     it->second->WakeUp();
 428 }
 429
 430 ThreadWatcherList::ThreadWatcherList() {
 431   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 432   CHECK(!g_thread_watcher_list_);
 433   g_thread_watcher_list_ = this;
 434 }
 435
 436 ThreadWatcherList::~ThreadWatcherList() {
 437   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 438   DCHECK(this == g_thread_watcher_list_);
 439   g_thread_watcher_list_ = NULL;
 440 }
 441
 442 // static
 443 void ThreadWatcherList::ParseCommandLine(
 444     const base::CommandLine& command_line,
 445     uint32* unresponsive_threshold,
 446     CrashOnHangThreadMap* crash_on_hang_threads) {
 447   // Initialize |unresponsive_threshold| to a default value.
 448   *unresponsive_threshold = kUnresponsiveCount;
 449
 450   // Increase the unresponsive_threshold on the Stable and Beta channels to
 451   // reduce the number of crashes due to ThreadWatcher.
 452   chrome::VersionInfo::Channel channel = chrome::VersionInfo::GetChannel();
 453   if (channel == chrome::VersionInfo::CHANNEL_STABLE) {
 454     *unresponsive_threshold *= 4;
 455   } else if (channel == chrome::VersionInfo::CHANNEL_BETA) {
 456     *unresponsive_threshold *= 2;
 457   }
 458
 459 #if defined(OS_WIN)
 460   // For Windows XP (old systems), double the unresponsive_threshold to give
 461   // the OS a chance to schedule UI/IO threads a time slice to respond with a
 462   // pong message (to get around limitations with the OS).
 463   if (base::win::GetVersion() <= base::win::VERSION_XP)
 464     *unresponsive_threshold *= 2;
 465 #endif
 466
 467   uint32 crash_seconds = *unresponsive_threshold * kUnresponsiveSeconds;
 468   std::string crash_on_hang_thread_names;
 469   if (command_line.HasSwitch(switches::kCrashOnHangThreads)) {
 470     crash_on_hang_thread_names =
 471         command_line.GetSwitchValueASCII(switches::kCrashOnHangThreads);
 472   } else if (channel != chrome::VersionInfo::CHANNEL_STABLE) {
 473     // Default to crashing the browser if UI or IO or FILE threads are not
 474     // responsive except in stable channel.
 475     crash_on_hang_thread_names = base::StringPrintf(
 476         "UI:%d:%d,IO:%d:%d,FILE:%d:%d",
 477         kLiveThreadsThreshold, crash_seconds,
 478         kLiveThreadsThreshold, crash_seconds,
 479         kLiveThreadsThreshold, crash_seconds * 5);
 480   }
 481
 482   ParseCommandLineCrashOnHangThreads(crash_on_hang_thread_names,
 483                                      kLiveThreadsThreshold,
 484                                      crash_seconds,
 485                                      crash_on_hang_threads);
 486 }
 487
 488 // static
 489 void ThreadWatcherList::ParseCommandLineCrashOnHangThreads(
 490     const std::string& crash_on_hang_thread_names,
 491     uint32 default_live_threads_threshold,
 492     uint32 default_crash_seconds,
 493     CrashOnHangThreadMap* crash_on_hang_threads) {
 494   base::StringTokenizer tokens(crash_on_hang_thread_names, ",");
 495   std::vector<std::string> values;
 496   while (tokens.GetNext()) {
 497     const std::string& token = tokens.token();
 498     base::SplitString(token, ':', &values);
 499     std::string thread_name = values[0];
 500
 501     uint32 live_threads_threshold = default_live_threads_threshold;
 502     uint32 crash_seconds = default_crash_seconds;
 503     if (values.size() >= 2 &&
 504         (!base::StringToUint(values[1], &live_threads_threshold))) {
 505       continue;
 506     }
 507     if (values.size() >= 3 &&
 508         (!base::StringToUint(values[2], &crash_seconds))) {
 509       continue;
 510     }
 511     uint32 unresponsive_threshold = static_cast<uint32>(
 512         ceil(static_cast<float>(crash_seconds) / kUnresponsiveSeconds));
 513
 514     CrashDataThresholds crash_data(live_threads_threshold,
 515                                    unresponsive_threshold);
 516     // Use the last specifier.
 517     (*crash_on_hang_threads)[thread_name] = crash_data;
 518   }
 519 }
 520
 521 // static
 522 void ThreadWatcherList::InitializeAndStartWatching(
 523     uint32 unresponsive_threshold,
 524     const CrashOnHangThreadMap& crash_on_hang_threads) {
 525   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 526
 527   // Disarm the startup timebomb, even if stop has been called.
 528   BrowserThread::PostTask(
 529       BrowserThread::UI,
 530       FROM_HERE,
 531       base::Bind(&StartupTimeBomb::DisarmStartupTimeBomb));
 532
 533   // This method is deferred in relationship to its StopWatchingAll()
 534   // counterpart. If a previous initialization has already happened, or if
 535   // stop has been called, there's nothing left to do here.
 536   if (g_thread_watcher_list_ || g_stopped_)
 537     return;
 538
 539   ThreadWatcherList* thread_watcher_list = new ThreadWatcherList();
 540   CHECK(thread_watcher_list);
 541
 542   const base::TimeDelta kSleepTime =
 543       base::TimeDelta::FromSeconds(kSleepSeconds);
 544   const base::TimeDelta kUnresponsiveTime =
 545       base::TimeDelta::FromSeconds(kUnresponsiveSeconds);
 546
 547   StartWatching(BrowserThread::UI, "UI", kSleepTime, kUnresponsiveTime,
 548                 unresponsive_threshold, crash_on_hang_threads);
 549   StartWatching(BrowserThread::IO, "IO", kSleepTime, kUnresponsiveTime,
 550                 unresponsive_threshold, crash_on_hang_threads);
 551   StartWatching(BrowserThread::DB, "DB", kSleepTime, kUnresponsiveTime,
 552                 unresponsive_threshold, crash_on_hang_threads);
 553   StartWatching(BrowserThread::FILE, "FILE", kSleepTime, kUnresponsiveTime,
 554                 unresponsive_threshold, crash_on_hang_threads);
 555   StartWatching(BrowserThread::CACHE, "CACHE", kSleepTime, kUnresponsiveTime,
 556                 unresponsive_threshold, crash_on_hang_threads);
 557 }
 558
 559 // static
 560 void ThreadWatcherList::StartWatching(
 561     const BrowserThread::ID& thread_id,
 562     const std::string& thread_name,
 563     const base::TimeDelta& sleep_time,
 564     const base::TimeDelta& unresponsive_time,
 565     uint32 unresponsive_threshold,
 566     const CrashOnHangThreadMap& crash_on_hang_threads) {
 567   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 568
 569   CrashOnHangThreadMap::const_iterator it =
 570       crash_on_hang_threads.find(thread_name);
 571   bool crash_on_hang = false;
 572   uint32 live_threads_threshold = 0;
 573   if (it != crash_on_hang_threads.end()) {
 574     crash_on_hang = true;
 575     live_threads_threshold = it->second.live_threads_threshold;
 576     unresponsive_threshold = it->second.unresponsive_threshold;
 577   }
 578
 579   ThreadWatcher::StartWatching(
 580       ThreadWatcher::WatchingParams(thread_id,
 581                                     thread_name,
 582                                     sleep_time,
 583                                     unresponsive_time,
 584                                     unresponsive_threshold,
 585                                     crash_on_hang,
 586                                     live_threads_threshold));
 587 }
 588
 589 // static
 590 void ThreadWatcherList::DeleteAll() {
 591   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
 592     WatchDogThread::PostTask(
 593         FROM_HERE,
 594         base::Bind(&ThreadWatcherList::DeleteAll));
 595     return;
 596   }
 597
 598   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 599
 600   SetStopped(true);
 601
 602   if (!g_thread_watcher_list_)
 603     return;
 604
 605   // Delete all thread watcher objects.
 606   while (!g_thread_watcher_list_->registered_.empty()) {
 607     RegistrationList::iterator it = g_thread_watcher_list_->registered_.begin();
 608     delete it->second;
 609     g_thread_watcher_list_->registered_.erase(it);
 610   }
 611
 612   delete g_thread_watcher_list_;
 613 }
 614
 615 // static
 616 ThreadWatcher* ThreadWatcherList::Find(const BrowserThread::ID& thread_id) {
 617   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 618   if (!g_thread_watcher_list_)
 619     return NULL;
 620   RegistrationList::iterator it =
 621       g_thread_watcher_list_->registered_.find(thread_id);
 622   if (g_thread_watcher_list_->registered_.end() == it)
 623     return NULL;
 624   return it->second;
 625 }
 626
 627 // static
 628 void ThreadWatcherList::SetStopped(bool stopped) {
 629   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 630   g_stopped_ = stopped;
 631 }
 632
 633 // ThreadWatcherObserver methods and members.
 634 //
 635 // static
 636 ThreadWatcherObserver* ThreadWatcherObserver::g_thread_watcher_observer_ = NULL;
 637
 638 ThreadWatcherObserver::ThreadWatcherObserver(
 639     const base::TimeDelta& wakeup_interval)
 640     : last_wakeup_time_(base::TimeTicks::Now()),
 641       wakeup_interval_(wakeup_interval) {
 642   CHECK(!g_thread_watcher_observer_);
 643   g_thread_watcher_observer_ = this;
 644 }
 645
 646 ThreadWatcherObserver::~ThreadWatcherObserver() {
 647   DCHECK(this == g_thread_watcher_observer_);
 648   g_thread_watcher_observer_ = NULL;
 649 }
 650
 651 // static
 652 void ThreadWatcherObserver::SetupNotifications(
 653     const base::TimeDelta& wakeup_interval) {
 654   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 655   ThreadWatcherObserver* observer = new ThreadWatcherObserver(wakeup_interval);
 656   observer->registrar_.Add(
 657       observer,
 658       chrome::NOTIFICATION_BROWSER_OPENED,
 659       content::NotificationService::AllBrowserContextsAndSources());
 660   observer->registrar_.Add(observer,
 661                            chrome::NOTIFICATION_BROWSER_CLOSED,
 662                            content::NotificationService::AllSources());
 663   observer->registrar_.Add(observer,
 664                            chrome::NOTIFICATION_TAB_PARENTED,
 665                            content::NotificationService::AllSources());
 666   observer->registrar_.Add(observer,
 667                            chrome::NOTIFICATION_TAB_CLOSING,
 668                            content::NotificationService::AllSources());
 669   observer->registrar_.Add(observer,
 670                            content::NOTIFICATION_LOAD_START,
 671                            content::NotificationService::AllSources());
 672   observer->registrar_.Add(observer,
 673                            content::NOTIFICATION_LOAD_STOP,
 674                            content::NotificationService::AllSources());
 675   observer->registrar_.Add(observer,
 676                            content::NOTIFICATION_RENDERER_PROCESS_CLOSED,
 677                            content::NotificationService::AllSources());
 678   observer->registrar_.Add(observer,
 679                            content::NOTIFICATION_RENDER_WIDGET_HOST_HANG,
 680                            content::NotificationService::AllSources());
 681   observer->registrar_.Add(observer,
 682                            chrome::NOTIFICATION_OMNIBOX_OPENED_URL,
 683                            content::NotificationService::AllSources());
 684 }
 685
 686 // static
 687 void ThreadWatcherObserver::RemoveNotifications() {
 688   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 689   if (!g_thread_watcher_observer_)
 690     return;
 691   g_thread_watcher_observer_->registrar_.RemoveAll();
 692   delete g_thread_watcher_observer_;
 693 }
 694
 695 void ThreadWatcherObserver::Observe(
 696     int type,
 697     const content::NotificationSource& source,
 698     const content::NotificationDetails& details) {
 699   // There is some user activity, see if thread watchers are to be awakened.
 700   base::TimeTicks now = base::TimeTicks::Now();
 701   if ((now - last_wakeup_time_) < wakeup_interval_)
 702     return;
 703   last_wakeup_time_ = now;
 704   WatchDogThread::PostTask(
 705       FROM_HERE,
 706       base::Bind(&ThreadWatcherList::WakeUpAll));
 707 }
 708
 709 // WatchDogThread methods and members.
 710
 711 // This lock protects g_watchdog_thread.
 712 static base::LazyInstance<base::Lock>::Leaky
 713     g_watchdog_lock = LAZY_INSTANCE_INITIALIZER;
 714
 715 // The singleton of this class.
 716 static WatchDogThread* g_watchdog_thread = NULL;
 717
 718 WatchDogThread::WatchDogThread() : Thread("BrowserWatchdog") {
 719 }
 720
 721 WatchDogThread::~WatchDogThread() {
 722   Stop();
 723 }
 724
 725 // static
 726 bool WatchDogThread::CurrentlyOnWatchDogThread() {
 727   base::AutoLock lock(g_watchdog_lock.Get());
 728   return g_watchdog_thread &&
 729       g_watchdog_thread->message_loop() == base::MessageLoop::current();
 730 }
 731
 732 // static
 733 bool WatchDogThread::PostTask(const tracked_objects::Location& from_here,
 734                               const base::Closure& task) {
 735   return PostTaskHelper(from_here, task, base::TimeDelta());
 736 }
 737
 738 // static
 739 bool WatchDogThread::PostDelayedTask(const tracked_objects::Location& from_here,
 740                                      const base::Closure& task,
 741                                      base::TimeDelta delay) {
 742   return PostTaskHelper(from_here, task, delay);
 743 }
 744
 745 // static
 746 bool WatchDogThread::PostTaskHelper(
 747     const tracked_objects::Location& from_here,
 748     const base::Closure& task,
 749     base::TimeDelta delay) {
 750   {
 751     base::AutoLock lock(g_watchdog_lock.Get());
 752
 753     base::MessageLoop* message_loop = g_watchdog_thread ?
 754         g_watchdog_thread->message_loop() : NULL;
 755     if (message_loop) {
 756       message_loop->PostDelayedTask(from_here, task, delay);
 757       return true;
 758     }
 759   }
 760
 761   return false;
 762 }
 763
 764 void WatchDogThread::Init() {
 765   // This thread shouldn't be allowed to perform any blocking disk I/O.
 766   base::ThreadRestrictions::SetIOAllowed(false);
 767
 768   base::AutoLock lock(g_watchdog_lock.Get());
 769   CHECK(!g_watchdog_thread);
 770   g_watchdog_thread = this;
 771 }
 772
 773 void WatchDogThread::CleanUp() {
 774   base::AutoLock lock(g_watchdog_lock.Get());
 775   g_watchdog_thread = NULL;
 776 }
 777
 778 namespace {
 779
 780 // StartupWatchDogThread methods and members.
 781 //
 782 // Class for detecting hangs during startup.
 783 class StartupWatchDogThread : public base::Watchdog {
 784  public:
 785   // Constructor specifies how long the StartupWatchDogThread will wait before
 786   // alarming.
 787   explicit StartupWatchDogThread(const base::TimeDelta& duration)
 788       : base::Watchdog(duration, "Startup watchdog thread", true) {
 789   }
 790
 791   // Alarm is called if the time expires after an Arm() without someone calling
 792   // Disarm(). When Alarm goes off, in release mode we get the crash dump
 793   // without crashing and in debug mode we break into the debugger.
 794   void Alarm() override {
 795 #if !defined(NDEBUG)
 796     metrics::StartupHang();
 797     return;
 798 #elif !defined(OS_ANDROID)
 799     WatchDogThread::PostTask(FROM_HERE, base::Bind(&metrics::StartupHang));
 800     return;
 801 #else
 802     // TODO(rtenneti): Enable crashing for Android.
 803 #endif  // OS_ANDROID
 804   }
 805
 806  private:
 807   DISALLOW_COPY_AND_ASSIGN(StartupWatchDogThread);
 808 };
 809
 810 // ShutdownWatchDogThread methods and members.
 811 //
 812 // Class for detecting hangs during shutdown.
 813 class ShutdownWatchDogThread : public base::Watchdog {
 814  public:
 815   // Constructor specifies how long the ShutdownWatchDogThread will wait before
 816   // alarming.
 817   explicit ShutdownWatchDogThread(const base::TimeDelta& duration)
 818       : base::Watchdog(duration, "Shutdown watchdog thread", true) {
 819   }
 820
 821   // Alarm is called if the time expires after an Arm() without someone calling
 822   // Disarm(). We crash the browser if this method is called.
 823   void Alarm() override { metrics::ShutdownHang(); }
 824
 825  private:
 826   DISALLOW_COPY_AND_ASSIGN(ShutdownWatchDogThread);
 827 };
 828 }  // namespace
 829
 830 // StartupTimeBomb methods and members.
 831 //
 832 // static
 833 StartupTimeBomb* StartupTimeBomb::g_startup_timebomb_ = NULL;
 834
 835 StartupTimeBomb::StartupTimeBomb()
 836     : startup_watchdog_(NULL),
 837       thread_id_(base::PlatformThread::CurrentId()) {
 838   CHECK(!g_startup_timebomb_);
 839   g_startup_timebomb_ = this;
 840 }
 841
 842 StartupTimeBomb::~StartupTimeBomb() {
 843   DCHECK(this == g_startup_timebomb_);
 844   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 845   if (startup_watchdog_)
 846     Disarm();
 847   g_startup_timebomb_ = NULL;
 848 }
 849
 850 void StartupTimeBomb::Arm(const base::TimeDelta& duration) {
 851   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 852   DCHECK(!startup_watchdog_);
 853   startup_watchdog_ = new StartupWatchDogThread(duration);
 854   startup_watchdog_->Arm();
 855   return;
 856 }
 857
 858 void StartupTimeBomb::Disarm() {
 859   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 860   if (startup_watchdog_) {
 861     startup_watchdog_->Disarm();
 862     startup_watchdog_->Cleanup();
 863     DeleteStartupWatchdog();
 864   }
 865 }
 866
 867 void StartupTimeBomb::DeleteStartupWatchdog() {
 868   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 869   if (startup_watchdog_->IsJoinable()) {
 870     // Allow the watchdog thread to shutdown on UI. Watchdog thread shutdowns
 871     // very fast.
 872     base::ThreadRestrictions::SetIOAllowed(true);
 873     delete startup_watchdog_;
 874     startup_watchdog_ = NULL;
 875     return;
 876   }
 877   base::MessageLoop::current()->PostDelayedTask(
 878       FROM_HERE,
 879       base::Bind(&StartupTimeBomb::DeleteStartupWatchdog,
 880                  base::Unretained(this)),
 881       base::TimeDelta::FromSeconds(10));
 882 }
 883
 884 // static
 885 void StartupTimeBomb::DisarmStartupTimeBomb() {
 886   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 887   if (g_startup_timebomb_)
 888     g_startup_timebomb_->Disarm();
 889 }
 890
 891 // ShutdownWatcherHelper methods and members.
 892 //
 893 // ShutdownWatcherHelper is a wrapper class for detecting hangs during
 894 // shutdown.
 895 ShutdownWatcherHelper::ShutdownWatcherHelper()
 896     : shutdown_watchdog_(NULL),
 897       thread_id_(base::PlatformThread::CurrentId()) {
 898 }
 899
 900 ShutdownWatcherHelper::~ShutdownWatcherHelper() {
 901   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 902   if (shutdown_watchdog_) {
 903     shutdown_watchdog_->Disarm();
 904     delete shutdown_watchdog_;
 905     shutdown_watchdog_ = NULL;
 906   }
 907 }
 908
 909 void ShutdownWatcherHelper::Arm(const base::TimeDelta& duration) {
 910   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 911   DCHECK(!shutdown_watchdog_);
 912   base::TimeDelta actual_duration = duration;
 913
 914   chrome::VersionInfo::Channel channel = chrome::VersionInfo::GetChannel();
 915   if (channel == chrome::VersionInfo::CHANNEL_STABLE) {
 916     actual_duration *= 20;
 917   } else if (channel == chrome::VersionInfo::CHANNEL_BETA ||
 918              channel == chrome::VersionInfo::CHANNEL_DEV) {
 919     actual_duration *= 10;
 920   }
 921
 922 #if defined(OS_WIN)
 923   // On Windows XP, give twice the time for shutdown.
 924   if (base::win::GetVersion() <= base::win::VERSION_XP)
 925     actual_duration *= 2;
 926 #endif
 927
 928   shutdown_watchdog_ = new ShutdownWatchDogThread(actual_duration);
 929   shutdown_watchdog_->Arm();
 930 }