chrome/browser/metrics/thread_watcher.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/metrics/thread_watcher.h"
   6
   7 #include <math.h>  // ceil
   8
   9 #include "base/bind.h"
  10 #include "base/compiler_specific.h"
  11 #include "base/debug/dump_without_crashing.h"
  12 #include "base/lazy_instance.h"
  13 #include "base/location.h"
  14 #include "base/strings/string_number_conversions.h"
  15 #include "base/strings/string_split.h"
  16 #include "base/strings/string_tokenizer.h"
  17 #include "base/strings/stringprintf.h"
  18 #include "base/thread_task_runner_handle.h"
  19 #include "base/threading/thread_restrictions.h"
  20 #include "build/build_config.h"
  21 #include "chrome/browser/chrome_notification_types.h"
  22 #include "chrome/browser/metrics/thread_watcher_report_hang.h"
  23 #include "chrome/common/chrome_switches.h"
  24 #include "chrome/common/chrome_version_info.h"
  25 #include "chrome/common/logging_chrome.h"
  26 #include "content/public/browser/notification_service.h"
  27
  28 #if defined(OS_WIN)
  29 #include "base/win/windows_version.h"
  30 #endif
  31
  32 using content::BrowserThread;
  33
  34 // ThreadWatcher methods and members.
  35 ThreadWatcher::ThreadWatcher(const WatchingParams& params)
  36     : thread_id_(params.thread_id),
  37       thread_name_(params.thread_name),
  38       watched_runner_(
  39           BrowserThread::GetMessageLoopProxyForThread(params.thread_id)),
  40       sleep_time_(params.sleep_time),
  41       unresponsive_time_(params.unresponsive_time),
  42       ping_time_(base::TimeTicks::Now()),
  43       pong_time_(ping_time_),
  44       ping_sequence_number_(0),
  45       active_(false),
  46       ping_count_(params.unresponsive_threshold),
  47       response_time_histogram_(NULL),
  48       unresponsive_time_histogram_(NULL),
  49       unresponsive_count_(0),
  50       hung_processing_complete_(false),
  51       unresponsive_threshold_(params.unresponsive_threshold),
  52       crash_on_hang_(params.crash_on_hang),
  53       live_threads_threshold_(params.live_threads_threshold),
  54       weak_ptr_factory_(this) {
  55   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  56   Initialize();
  57 }
  58
  59 ThreadWatcher::~ThreadWatcher() {}
  60
  61 // static
  62 void ThreadWatcher::StartWatching(const WatchingParams& params) {
  63   DCHECK_GE(params.sleep_time.InMilliseconds(), 0);
  64   DCHECK_GE(params.unresponsive_time.InMilliseconds(),
  65             params.sleep_time.InMilliseconds());
  66
  67   // If we are not on WatchDogThread, then post a task to call StartWatching on
  68   // WatchDogThread.
  69   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
  70     WatchDogThread::PostTask(
  71         FROM_HERE,
  72         base::Bind(&ThreadWatcher::StartWatching, params));
  73     return;
  74   }
  75
  76   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  77
  78   // Create a new thread watcher object for the given thread and activate it.
  79   ThreadWatcher* watcher = new ThreadWatcher(params);
  80
  81   DCHECK(watcher);
  82   // If we couldn't register the thread watcher object, we are shutting down,
  83   // then don't activate thread watching.
  84   if (!ThreadWatcherList::IsRegistered(params.thread_id))
  85     return;
  86   watcher->ActivateThreadWatching();
  87 }
  88
  89 void ThreadWatcher::ActivateThreadWatching() {
  90   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  91   if (active_) return;
  92   active_ = true;
  93   ping_count_ = unresponsive_threshold_;
  94   ResetHangCounters();
  95   base::ThreadTaskRunnerHandle::Get()->PostTask(
  96       FROM_HERE, base::Bind(&ThreadWatcher::PostPingMessage,
  97                             weak_ptr_factory_.GetWeakPtr()));
  98 }
  99
 100 void ThreadWatcher::DeActivateThreadWatching() {
 101   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 102   active_ = false;
 103   ping_count_ = 0;
 104   weak_ptr_factory_.InvalidateWeakPtrs();
 105 }
 106
 107 void ThreadWatcher::WakeUp() {
 108   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 109   // There is some user activity, PostPingMessage task of thread watcher if
 110   // needed.
 111   if (!active_) return;
 112
 113   // Throw away the previous |unresponsive_count_| and start over again. Just
 114   // before going to sleep, |unresponsive_count_| could be very close to
 115   // |unresponsive_threshold_| and when user becomes active,
 116   // |unresponsive_count_| can go over |unresponsive_threshold_| if there was no
 117   // response for ping messages. Reset |unresponsive_count_| to start measuring
 118   // the unresponsiveness of the threads when system becomes active.
 119   unresponsive_count_ = 0;
 120
 121   if (ping_count_ <= 0) {
 122     ping_count_ = unresponsive_threshold_;
 123     ResetHangCounters();
 124     PostPingMessage();
 125   } else {
 126     ping_count_ = unresponsive_threshold_;
 127   }
 128 }
 129
 130 void ThreadWatcher::PostPingMessage() {
 131   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 132   // If we have stopped watching or if the user is idle, then stop sending
 133   // ping messages.
 134   if (!active_ || ping_count_ <= 0)
 135     return;
 136
 137   // Save the current time when we have sent ping message.
 138   ping_time_ = base::TimeTicks::Now();
 139
 140   // Send a ping message to the watched thread. Callback will be called on
 141   // the WatchDogThread.
 142   base::Closure callback(
 143       base::Bind(&ThreadWatcher::OnPongMessage, weak_ptr_factory_.GetWeakPtr(),
 144                  ping_sequence_number_));
 145   if (watched_runner_->PostTask(
 146           FROM_HERE,
 147           base::Bind(&ThreadWatcher::OnPingMessage, thread_id_,
 148                      callback))) {
 149       // Post a task to check the responsiveness of watched thread.
 150       base::MessageLoop::current()->PostDelayedTask(
 151           FROM_HERE,
 152           base::Bind(&ThreadWatcher::OnCheckResponsiveness,
 153                      weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
 154           unresponsive_time_);
 155   } else {
 156     // Watched thread might have gone away, stop watching it.
 157     DeActivateThreadWatching();
 158   }
 159 }
 160
 161 void ThreadWatcher::OnPongMessage(uint64 ping_sequence_number) {
 162   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 163
 164   // Record watched thread's response time.
 165   base::TimeTicks now = base::TimeTicks::Now();
 166   base::TimeDelta response_time = now - ping_time_;
 167   response_time_histogram_->AddTime(response_time);
 168
 169   // Save the current time when we have got pong message.
 170   pong_time_ = now;
 171
 172   // Check if there are any extra pings in flight.
 173   DCHECK_EQ(ping_sequence_number_, ping_sequence_number);
 174   if (ping_sequence_number_ != ping_sequence_number)
 175     return;
 176
 177   // Increment sequence number for the next ping message to indicate watched
 178   // thread is responsive.
 179   ++ping_sequence_number_;
 180
 181   // If we have stopped watching or if the user is idle, then stop sending
 182   // ping messages.
 183   if (!active_ || --ping_count_ <= 0)
 184     return;
 185
 186   base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 187       FROM_HERE, base::Bind(&ThreadWatcher::PostPingMessage,
 188                             weak_ptr_factory_.GetWeakPtr()),
 189       sleep_time_);
 190 }
 191
 192 void ThreadWatcher::OnCheckResponsiveness(uint64 ping_sequence_number) {
 193   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 194   // If we have stopped watching then consider thread as responding.
 195   if (!active_) {
 196     responsive_ = true;
 197     return;
 198   }
 199   // If the latest ping_sequence_number_ is not same as the ping_sequence_number
 200   // that is passed in, then we can assume OnPongMessage was called.
 201   // OnPongMessage increments ping_sequence_number_.
 202   if (ping_sequence_number_ != ping_sequence_number) {
 203     // Reset unresponsive_count_ to zero because we got a response from the
 204     // watched thread.
 205     ResetHangCounters();
 206
 207     responsive_ = true;
 208     return;
 209   }
 210   // Record that we got no response from watched thread.
 211   GotNoResponse();
 212
 213   // Post a task to check the responsiveness of watched thread.
 214   base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 215       FROM_HERE,
 216       base::Bind(&ThreadWatcher::OnCheckResponsiveness,
 217                  weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
 218       unresponsive_time_);
 219   responsive_ = false;
 220 }
 221
 222 void ThreadWatcher::Initialize() {
 223   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 224   ThreadWatcherList::Register(this);
 225
 226   const std::string response_time_histogram_name =
 227       "ThreadWatcher.ResponseTime." + thread_name_;
 228   response_time_histogram_ = base::Histogram::FactoryTimeGet(
 229       response_time_histogram_name,
 230       base::TimeDelta::FromMilliseconds(1),
 231       base::TimeDelta::FromSeconds(100), 50,
 232       base::Histogram::kUmaTargetedHistogramFlag);
 233
 234   const std::string unresponsive_time_histogram_name =
 235       "ThreadWatcher.Unresponsive." + thread_name_;
 236   unresponsive_time_histogram_ = base::Histogram::FactoryTimeGet(
 237       unresponsive_time_histogram_name,
 238       base::TimeDelta::FromMilliseconds(1),
 239       base::TimeDelta::FromSeconds(100), 50,
 240       base::Histogram::kUmaTargetedHistogramFlag);
 241
 242   const std::string responsive_count_histogram_name =
 243       "ThreadWatcher.ResponsiveThreads." + thread_name_;
 244   responsive_count_histogram_ = base::LinearHistogram::FactoryGet(
 245       responsive_count_histogram_name, 1, 10, 11,
 246       base::Histogram::kUmaTargetedHistogramFlag);
 247
 248   const std::string unresponsive_count_histogram_name =
 249       "ThreadWatcher.UnresponsiveThreads." + thread_name_;
 250   unresponsive_count_histogram_ = base::LinearHistogram::FactoryGet(
 251       unresponsive_count_histogram_name, 1, 10, 11,
 252       base::Histogram::kUmaTargetedHistogramFlag);
 253 }
 254
 255 // static
 256 void ThreadWatcher::OnPingMessage(const BrowserThread::ID& thread_id,
 257                                   const base::Closure& callback_task) {
 258   // This method is called on watched thread.
 259   DCHECK(BrowserThread::CurrentlyOn(thread_id));
 260   WatchDogThread::PostTask(FROM_HERE, callback_task);
 261 }
 262
 263 void ThreadWatcher::ResetHangCounters() {
 264   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 265   unresponsive_count_ = 0;
 266   hung_processing_complete_ = false;
 267 }
 268
 269 void ThreadWatcher::GotNoResponse() {
 270   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 271
 272   ++unresponsive_count_;
 273   if (!IsVeryUnresponsive())
 274     return;
 275
 276   // Record total unresponsive_time since last pong message.
 277   base::TimeDelta unresponse_time = base::TimeTicks::Now() - pong_time_;
 278   unresponsive_time_histogram_->AddTime(unresponse_time);
 279
 280   // We have already collected stats for the non-responding watched thread.
 281   if (hung_processing_complete_)
 282     return;
 283
 284   // Record how other threads are responding.
 285   uint32 responding_thread_count = 0;
 286   uint32 unresponding_thread_count = 0;
 287   ThreadWatcherList::GetStatusOfThreads(&responding_thread_count,
 288                                         &unresponding_thread_count);
 289
 290   // Record how many watched threads are responding.
 291   responsive_count_histogram_->Add(responding_thread_count);
 292
 293   // Record how many watched threads are not responding.
 294   unresponsive_count_histogram_->Add(unresponding_thread_count);
 295
 296   // Crash the browser if the watched thread is to be crashed on hang and if the
 297   // number of other threads responding is less than or equal to
 298   // live_threads_threshold_ and at least one other thread is responding.
 299   if (crash_on_hang_ &&
 300       responding_thread_count > 0 &&
 301       responding_thread_count <= live_threads_threshold_) {
 302     static bool crashed_once = false;
 303     if (!crashed_once) {
 304       crashed_once = true;
 305       metrics::CrashBecauseThreadWasUnresponsive(thread_id_);
 306     }
 307   }
 308
 309   hung_processing_complete_ = true;
 310 }
 311
 312 bool ThreadWatcher::IsVeryUnresponsive() {
 313   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 314   return unresponsive_count_ >= unresponsive_threshold_;
 315 }
 316
 317 // ThreadWatcherList methods and members.
 318 //
 319 // static
 320 ThreadWatcherList* ThreadWatcherList::g_thread_watcher_list_ = NULL;
 321 // static
 322 bool ThreadWatcherList::g_stopped_ = false;
 323 // static
 324 const int ThreadWatcherList::kSleepSeconds = 1;
 325 // static
 326 const int ThreadWatcherList::kUnresponsiveSeconds = 2;
 327 // static
 328 const int ThreadWatcherList::kUnresponsiveCount = 9;
 329 // static
 330 const int ThreadWatcherList::kLiveThreadsThreshold = 2;
 331 // static, non-const for tests.
 332 int ThreadWatcherList::g_initialize_delay_seconds = 120;
 333
 334 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds(
 335     uint32 live_threads_threshold,
 336     uint32 unresponsive_threshold)
 337     : live_threads_threshold(live_threads_threshold),
 338       unresponsive_threshold(unresponsive_threshold) {
 339 }
 340
 341 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds()
 342     : live_threads_threshold(kLiveThreadsThreshold),
 343       unresponsive_threshold(kUnresponsiveCount) {
 344 }
 345
 346 // static
 347 void ThreadWatcherList::StartWatchingAll(
 348     const base::CommandLine& command_line) {
 349   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 350   uint32 unresponsive_threshold;
 351   CrashOnHangThreadMap crash_on_hang_threads;
 352   ParseCommandLine(command_line,
 353                    &unresponsive_threshold,
 354                    &crash_on_hang_threads);
 355
 356   ThreadWatcherObserver::SetupNotifications(
 357       base::TimeDelta::FromSeconds(kSleepSeconds * unresponsive_threshold));
 358
 359   WatchDogThread::PostTask(
 360       FROM_HERE,
 361       base::Bind(&ThreadWatcherList::SetStopped, false));
 362
 363   if (!WatchDogThread::PostDelayedTask(
 364           FROM_HERE,
 365           base::Bind(&ThreadWatcherList::InitializeAndStartWatching,
 366                      unresponsive_threshold,
 367                      crash_on_hang_threads),
 368           base::TimeDelta::FromSeconds(g_initialize_delay_seconds))) {
 369     // Disarm() the startup timebomb, if we couldn't post the task to start the
 370     // ThreadWatcher (becasue WatchDog thread is not running).
 371     StartupTimeBomb::DisarmStartupTimeBomb();
 372   }
 373 }
 374
 375 // static
 376 void ThreadWatcherList::StopWatchingAll() {
 377   // TODO(rtenneti): Enable ThreadWatcher.
 378   ThreadWatcherObserver::RemoveNotifications();
 379   DeleteAll();
 380 }
 381
 382 // static
 383 void ThreadWatcherList::Register(ThreadWatcher* watcher) {
 384   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 385   if (!g_thread_watcher_list_)
 386     return;
 387   DCHECK(!g_thread_watcher_list_->Find(watcher->thread_id()));
 388   g_thread_watcher_list_->registered_[watcher->thread_id()] = watcher;
 389 }
 390
 391 // static
 392 bool ThreadWatcherList::IsRegistered(const BrowserThread::ID thread_id) {
 393   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 394   return NULL != ThreadWatcherList::Find(thread_id);
 395 }
 396
 397 // static
 398 void ThreadWatcherList::GetStatusOfThreads(uint32* responding_thread_count,
 399                                            uint32* unresponding_thread_count) {
 400   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 401   *responding_thread_count = 0;
 402   *unresponding_thread_count = 0;
 403   if (!g_thread_watcher_list_)
 404     return;
 405
 406   for (RegistrationList::iterator it =
 407            g_thread_watcher_list_->registered_.begin();
 408        g_thread_watcher_list_->registered_.end() != it;
 409        ++it) {
 410     if (it->second->IsVeryUnresponsive())
 411       ++(*unresponding_thread_count);
 412     else
 413       ++(*responding_thread_count);
 414   }
 415 }
 416
 417 // static
 418 void ThreadWatcherList::WakeUpAll() {
 419   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 420   if (!g_thread_watcher_list_)
 421     return;
 422
 423   for (RegistrationList::iterator it =
 424            g_thread_watcher_list_->registered_.begin();
 425        g_thread_watcher_list_->registered_.end() != it;
 426        ++it)
 427     it->second->WakeUp();
 428 }
 429
 430 ThreadWatcherList::ThreadWatcherList() {
 431   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 432   CHECK(!g_thread_watcher_list_);
 433   g_thread_watcher_list_ = this;
 434 }
 435
 436 ThreadWatcherList::~ThreadWatcherList() {
 437   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 438   DCHECK(this == g_thread_watcher_list_);
 439   g_thread_watcher_list_ = NULL;
 440 }
 441
 442 // static
 443 void ThreadWatcherList::ParseCommandLine(
 444     const base::CommandLine& command_line,
 445     uint32* unresponsive_threshold,
 446     CrashOnHangThreadMap* crash_on_hang_threads) {
 447   // Initialize |unresponsive_threshold| to a default value.
 448   *unresponsive_threshold = kUnresponsiveCount;
 449
 450   // Increase the unresponsive_threshold on the Stable and Beta channels to
 451   // reduce the number of crashes due to ThreadWatcher.
 452   chrome::VersionInfo::Channel channel = chrome::VersionInfo::GetChannel();
 453   if (channel == chrome::VersionInfo::CHANNEL_STABLE) {
 454     *unresponsive_threshold *= 4;
 455   } else if (channel == chrome::VersionInfo::CHANNEL_BETA) {
 456     *unresponsive_threshold *= 2;
 457   }
 458
 459 #if defined(OS_WIN)
 460   // For Windows XP (old systems), double the unresponsive_threshold to give
 461   // the OS a chance to schedule UI/IO threads a time slice to respond with a
 462   // pong message (to get around limitations with the OS).
 463   if (base::win::GetVersion() <= base::win::VERSION_XP)
 464     *unresponsive_threshold *= 2;
 465 #endif
 466
 467   uint32 crash_seconds = *unresponsive_threshold * kUnresponsiveSeconds;
 468   std::string crash_on_hang_thread_names;
 469   if (command_line.HasSwitch(switches::kCrashOnHangThreads)) {
 470     crash_on_hang_thread_names =
 471         command_line.GetSwitchValueASCII(switches::kCrashOnHangThreads);
 472   } else if (channel != chrome::VersionInfo::CHANNEL_STABLE) {
 473     // Default to crashing the browser if UI or IO or FILE threads are not
 474     // responsive except in stable channel.
 475     crash_on_hang_thread_names = base::StringPrintf(
 476         "UI:%d:%d,IO:%d:%d,FILE:%d:%d",
 477         kLiveThreadsThreshold, crash_seconds,
 478         kLiveThreadsThreshold, crash_seconds,
 479         kLiveThreadsThreshold, crash_seconds * 5);
 480   }
 481
 482   ParseCommandLineCrashOnHangThreads(crash_on_hang_thread_names,
 483                                      kLiveThreadsThreshold,
 484                                      crash_seconds,
 485                                      crash_on_hang_threads);
 486 }
 487
 488 // static
 489 void ThreadWatcherList::ParseCommandLineCrashOnHangThreads(
 490     const std::string& crash_on_hang_thread_names,
 491     uint32 default_live_threads_threshold,
 492     uint32 default_crash_seconds,
 493     CrashOnHangThreadMap* crash_on_hang_threads) {
 494   base::StringTokenizer tokens(crash_on_hang_thread_names, ",");
 495   std::vector<std::string> values;
 496   while (tokens.GetNext()) {
 497     const std::string& token = tokens.token();
 498     base::SplitString(token, ':', &values);
 499     std::string thread_name = values[0];
 500
 501     uint32 live_threads_threshold = default_live_threads_threshold;
 502     uint32 crash_seconds = default_crash_seconds;
 503     if (values.size() >= 2 &&
 504         (!base::StringToUint(values[1], &live_threads_threshold))) {
 505       continue;
 506     }
 507     if (values.size() >= 3 &&
 508         (!base::StringToUint(values[2], &crash_seconds))) {
 509       continue;
 510     }
 511     uint32 unresponsive_threshold = static_cast<uint32>(
 512         ceil(static_cast<float>(crash_seconds) / kUnresponsiveSeconds));
 513
 514     CrashDataThresholds crash_data(live_threads_threshold,
 515                                    unresponsive_threshold);
 516     // Use the last specifier.
 517     (*crash_on_hang_threads)[thread_name] = crash_data;
 518   }
 519 }
 520
 521 // static
 522 void ThreadWatcherList::InitializeAndStartWatching(
 523     uint32 unresponsive_threshold,
 524     const CrashOnHangThreadMap& crash_on_hang_threads) {
 525   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 526
 527   // Disarm the startup timebomb, even if stop has been called.
 528   BrowserThread::PostTask(
 529       BrowserThread::UI,
 530       FROM_HERE,
 531       base::Bind(&StartupTimeBomb::DisarmStartupTimeBomb));
 532
 533   // This method is deferred in relationship to its StopWatchingAll()
 534   // counterpart. If a previous initialization has already happened, or if
 535   // stop has been called, there's nothing left to do here.
 536   if (g_thread_watcher_list_ || g_stopped_)
 537     return;
 538
 539   ThreadWatcherList* thread_watcher_list = new ThreadWatcherList();
 540   CHECK(thread_watcher_list);
 541
 542   // TODO(rtenneti): Because we don't generate crash dumps for ThreadWatcher in
 543   // stable channel, disable ThreadWatcher in stable and unknown channels. We
 544   // will also not collect histogram data in these channels until
 545   // http://crbug.com/426203 is fixed.
 546   chrome::VersionInfo::Channel channel = chrome::VersionInfo::GetChannel();
 547   if (channel == chrome::VersionInfo::CHANNEL_STABLE ||
 548       channel == chrome::VersionInfo::CHANNEL_UNKNOWN) {
 549     return;
 550   }
 551
 552   const base::TimeDelta kSleepTime =
 553       base::TimeDelta::FromSeconds(kSleepSeconds);
 554   const base::TimeDelta kUnresponsiveTime =
 555       base::TimeDelta::FromSeconds(kUnresponsiveSeconds);
 556
 557   StartWatching(BrowserThread::UI, "UI", kSleepTime, kUnresponsiveTime,
 558                 unresponsive_threshold, crash_on_hang_threads);
 559   StartWatching(BrowserThread::IO, "IO", kSleepTime, kUnresponsiveTime,
 560                 unresponsive_threshold, crash_on_hang_threads);
 561   StartWatching(BrowserThread::DB, "DB", kSleepTime, kUnresponsiveTime,
 562                 unresponsive_threshold, crash_on_hang_threads);
 563   StartWatching(BrowserThread::FILE, "FILE", kSleepTime, kUnresponsiveTime,
 564                 unresponsive_threshold, crash_on_hang_threads);
 565   StartWatching(BrowserThread::CACHE, "CACHE", kSleepTime, kUnresponsiveTime,
 566                 unresponsive_threshold, crash_on_hang_threads);
 567 }
 568
 569 // static
 570 void ThreadWatcherList::StartWatching(
 571     const BrowserThread::ID& thread_id,
 572     const std::string& thread_name,
 573     const base::TimeDelta& sleep_time,
 574     const base::TimeDelta& unresponsive_time,
 575     uint32 unresponsive_threshold,
 576     const CrashOnHangThreadMap& crash_on_hang_threads) {
 577   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 578
 579   CrashOnHangThreadMap::const_iterator it =
 580       crash_on_hang_threads.find(thread_name);
 581   bool crash_on_hang = false;
 582   uint32 live_threads_threshold = 0;
 583   if (it != crash_on_hang_threads.end()) {
 584     crash_on_hang = true;
 585     live_threads_threshold = it->second.live_threads_threshold;
 586     unresponsive_threshold = it->second.unresponsive_threshold;
 587   }
 588
 589   ThreadWatcher::StartWatching(
 590       ThreadWatcher::WatchingParams(thread_id,
 591                                     thread_name,
 592                                     sleep_time,
 593                                     unresponsive_time,
 594                                     unresponsive_threshold,
 595                                     crash_on_hang,
 596                                     live_threads_threshold));
 597 }
 598
 599 // static
 600 void ThreadWatcherList::DeleteAll() {
 601   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
 602     WatchDogThread::PostTask(
 603         FROM_HERE,
 604         base::Bind(&ThreadWatcherList::DeleteAll));
 605     return;
 606   }
 607
 608   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 609
 610   SetStopped(true);
 611
 612   if (!g_thread_watcher_list_)
 613     return;
 614
 615   // Delete all thread watcher objects.
 616   while (!g_thread_watcher_list_->registered_.empty()) {
 617     RegistrationList::iterator it = g_thread_watcher_list_->registered_.begin();
 618     delete it->second;
 619     g_thread_watcher_list_->registered_.erase(it);
 620   }
 621
 622   delete g_thread_watcher_list_;
 623 }
 624
 625 // static
 626 ThreadWatcher* ThreadWatcherList::Find(const BrowserThread::ID& thread_id) {
 627   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 628   if (!g_thread_watcher_list_)
 629     return NULL;
 630   RegistrationList::iterator it =
 631       g_thread_watcher_list_->registered_.find(thread_id);
 632   if (g_thread_watcher_list_->registered_.end() == it)
 633     return NULL;
 634   return it->second;
 635 }
 636
 637 // static
 638 void ThreadWatcherList::SetStopped(bool stopped) {
 639   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 640   g_stopped_ = stopped;
 641 }
 642
 643 // ThreadWatcherObserver methods and members.
 644 //
 645 // static
 646 ThreadWatcherObserver* ThreadWatcherObserver::g_thread_watcher_observer_ = NULL;
 647
 648 ThreadWatcherObserver::ThreadWatcherObserver(
 649     const base::TimeDelta& wakeup_interval)
 650     : last_wakeup_time_(base::TimeTicks::Now()),
 651       wakeup_interval_(wakeup_interval) {
 652   CHECK(!g_thread_watcher_observer_);
 653   g_thread_watcher_observer_ = this;
 654 }
 655
 656 ThreadWatcherObserver::~ThreadWatcherObserver() {
 657   DCHECK(this == g_thread_watcher_observer_);
 658   g_thread_watcher_observer_ = NULL;
 659 }
 660
 661 // static
 662 void ThreadWatcherObserver::SetupNotifications(
 663     const base::TimeDelta& wakeup_interval) {
 664   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 665   ThreadWatcherObserver* observer = new ThreadWatcherObserver(wakeup_interval);
 666   observer->registrar_.Add(
 667       observer,
 668       chrome::NOTIFICATION_BROWSER_OPENED,
 669       content::NotificationService::AllBrowserContextsAndSources());
 670   observer->registrar_.Add(observer,
 671                            chrome::NOTIFICATION_BROWSER_CLOSED,
 672                            content::NotificationService::AllSources());
 673   observer->registrar_.Add(observer,
 674                            chrome::NOTIFICATION_TAB_PARENTED,
 675                            content::NotificationService::AllSources());
 676   observer->registrar_.Add(observer,
 677                            chrome::NOTIFICATION_TAB_CLOSING,
 678                            content::NotificationService::AllSources());
 679   observer->registrar_.Add(observer,
 680                            content::NOTIFICATION_LOAD_START,
 681                            content::NotificationService::AllSources());
 682   observer->registrar_.Add(observer,
 683                            content::NOTIFICATION_LOAD_STOP,
 684                            content::NotificationService::AllSources());
 685   observer->registrar_.Add(observer,
 686                            content::NOTIFICATION_RENDERER_PROCESS_CLOSED,
 687                            content::NotificationService::AllSources());
 688   observer->registrar_.Add(observer,
 689                            content::NOTIFICATION_RENDER_WIDGET_HOST_HANG,
 690                            content::NotificationService::AllSources());
 691   observer->registrar_.Add(observer,
 692                            chrome::NOTIFICATION_OMNIBOX_OPENED_URL,
 693                            content::NotificationService::AllSources());
 694 }
 695
 696 // static
 697 void ThreadWatcherObserver::RemoveNotifications() {
 698   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 699   if (!g_thread_watcher_observer_)
 700     return;
 701   g_thread_watcher_observer_->registrar_.RemoveAll();
 702   delete g_thread_watcher_observer_;
 703 }
 704
 705 void ThreadWatcherObserver::Observe(
 706     int type,
 707     const content::NotificationSource& source,
 708     const content::NotificationDetails& details) {
 709   // There is some user activity, see if thread watchers are to be awakened.
 710   base::TimeTicks now = base::TimeTicks::Now();
 711   if ((now - last_wakeup_time_) < wakeup_interval_)
 712     return;
 713   last_wakeup_time_ = now;
 714   WatchDogThread::PostTask(
 715       FROM_HERE,
 716       base::Bind(&ThreadWatcherList::WakeUpAll));
 717 }
 718
 719 // WatchDogThread methods and members.
 720
 721 // This lock protects g_watchdog_thread.
 722 static base::LazyInstance<base::Lock>::Leaky
 723     g_watchdog_lock = LAZY_INSTANCE_INITIALIZER;
 724
 725 // The singleton of this class.
 726 static WatchDogThread* g_watchdog_thread = NULL;
 727
 728 WatchDogThread::WatchDogThread() : Thread("BrowserWatchdog") {
 729 }
 730
 731 WatchDogThread::~WatchDogThread() {
 732   Stop();
 733 }
 734
 735 // static
 736 bool WatchDogThread::CurrentlyOnWatchDogThread() {
 737   base::AutoLock lock(g_watchdog_lock.Get());
 738   return g_watchdog_thread &&
 739       g_watchdog_thread->message_loop() == base::MessageLoop::current();
 740 }
 741
 742 // static
 743 bool WatchDogThread::PostTask(const tracked_objects::Location& from_here,
 744                               const base::Closure& task) {
 745   return PostTaskHelper(from_here, task, base::TimeDelta());
 746 }
 747
 748 // static
 749 bool WatchDogThread::PostDelayedTask(const tracked_objects::Location& from_here,
 750                                      const base::Closure& task,
 751                                      base::TimeDelta delay) {
 752   return PostTaskHelper(from_here, task, delay);
 753 }
 754
 755 // static
 756 bool WatchDogThread::PostTaskHelper(
 757     const tracked_objects::Location& from_here,
 758     const base::Closure& task,
 759     base::TimeDelta delay) {
 760   {
 761     base::AutoLock lock(g_watchdog_lock.Get());
 762
 763     base::MessageLoop* message_loop = g_watchdog_thread ?
 764         g_watchdog_thread->message_loop() : NULL;
 765     if (message_loop) {
 766       message_loop->task_runner()->PostDelayedTask(from_here, task, delay);
 767       return true;
 768     }
 769   }
 770
 771   return false;
 772 }
 773
 774 void WatchDogThread::Init() {
 775   // This thread shouldn't be allowed to perform any blocking disk I/O.
 776   base::ThreadRestrictions::SetIOAllowed(false);
 777
 778   base::AutoLock lock(g_watchdog_lock.Get());
 779   CHECK(!g_watchdog_thread);
 780   g_watchdog_thread = this;
 781 }
 782
 783 void WatchDogThread::CleanUp() {
 784   base::AutoLock lock(g_watchdog_lock.Get());
 785   g_watchdog_thread = NULL;
 786 }
 787
 788 namespace {
 789
 790 // StartupWatchDogThread methods and members.
 791 //
 792 // Class for detecting hangs during startup.
 793 class StartupWatchDogThread : public base::Watchdog {
 794  public:
 795   // Constructor specifies how long the StartupWatchDogThread will wait before
 796   // alarming.
 797   explicit StartupWatchDogThread(const base::TimeDelta& duration)
 798       : base::Watchdog(duration, "Startup watchdog thread", true) {
 799   }
 800
 801   // Alarm is called if the time expires after an Arm() without someone calling
 802   // Disarm(). When Alarm goes off, in release mode we get the crash dump
 803   // without crashing and in debug mode we break into the debugger.
 804   void Alarm() override {
 805 #if !defined(NDEBUG)
 806     metrics::StartupHang();
 807     return;
 808 #elif !defined(OS_ANDROID)
 809     WatchDogThread::PostTask(FROM_HERE, base::Bind(&metrics::StartupHang));
 810     return;
 811 #else
 812     // TODO(rtenneti): Enable crashing for Android.
 813 #endif  // OS_ANDROID
 814   }
 815
 816  private:
 817   DISALLOW_COPY_AND_ASSIGN(StartupWatchDogThread);
 818 };
 819
 820 // ShutdownWatchDogThread methods and members.
 821 //
 822 // Class for detecting hangs during shutdown.
 823 class ShutdownWatchDogThread : public base::Watchdog {
 824  public:
 825   // Constructor specifies how long the ShutdownWatchDogThread will wait before
 826   // alarming.
 827   explicit ShutdownWatchDogThread(const base::TimeDelta& duration)
 828       : base::Watchdog(duration, "Shutdown watchdog thread", true) {
 829   }
 830
 831   // Alarm is called if the time expires after an Arm() without someone calling
 832   // Disarm(). We crash the browser if this method is called.
 833   void Alarm() override { metrics::ShutdownHang(); }
 834
 835  private:
 836   DISALLOW_COPY_AND_ASSIGN(ShutdownWatchDogThread);
 837 };
 838 }  // namespace
 839
 840 // StartupTimeBomb methods and members.
 841 //
 842 // static
 843 StartupTimeBomb* StartupTimeBomb::g_startup_timebomb_ = NULL;
 844
 845 StartupTimeBomb::StartupTimeBomb()
 846     : startup_watchdog_(NULL),
 847       thread_id_(base::PlatformThread::CurrentId()) {
 848   CHECK(!g_startup_timebomb_);
 849   g_startup_timebomb_ = this;
 850 }
 851
 852 StartupTimeBomb::~StartupTimeBomb() {
 853   DCHECK(this == g_startup_timebomb_);
 854   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 855   if (startup_watchdog_)
 856     Disarm();
 857   g_startup_timebomb_ = NULL;
 858 }
 859
 860 void StartupTimeBomb::Arm(const base::TimeDelta& duration) {
 861   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 862   DCHECK(!startup_watchdog_);
 863   startup_watchdog_ = new StartupWatchDogThread(duration);
 864   startup_watchdog_->Arm();
 865   return;
 866 }
 867
 868 void StartupTimeBomb::Disarm() {
 869   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 870   if (startup_watchdog_) {
 871     startup_watchdog_->Disarm();
 872     startup_watchdog_->Cleanup();
 873     DeleteStartupWatchdog();
 874   }
 875 }
 876
 877 void StartupTimeBomb::DeleteStartupWatchdog() {
 878   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 879   if (startup_watchdog_->IsJoinable()) {
 880     // Allow the watchdog thread to shutdown on UI. Watchdog thread shutdowns
 881     // very fast.
 882     base::ThreadRestrictions::SetIOAllowed(true);
 883     delete startup_watchdog_;
 884     startup_watchdog_ = NULL;
 885     return;
 886   }
 887   base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 888       FROM_HERE, base::Bind(&StartupTimeBomb::DeleteStartupWatchdog,
 889                             base::Unretained(this)),
 890       base::TimeDelta::FromSeconds(10));
 891 }
 892
 893 // static
 894 void StartupTimeBomb::DisarmStartupTimeBomb() {
 895   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 896   if (g_startup_timebomb_)
 897     g_startup_timebomb_->Disarm();
 898 }
 899
 900 // ShutdownWatcherHelper methods and members.
 901 //
 902 // ShutdownWatcherHelper is a wrapper class for detecting hangs during
 903 // shutdown.
 904 ShutdownWatcherHelper::ShutdownWatcherHelper()
 905     : shutdown_watchdog_(NULL),
 906       thread_id_(base::PlatformThread::CurrentId()) {
 907 }
 908
 909 ShutdownWatcherHelper::~ShutdownWatcherHelper() {
 910   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 911   if (shutdown_watchdog_) {
 912     shutdown_watchdog_->Disarm();
 913     delete shutdown_watchdog_;
 914     shutdown_watchdog_ = NULL;
 915   }
 916 }
 917
 918 void ShutdownWatcherHelper::Arm(const base::TimeDelta& duration) {
 919   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 920   DCHECK(!shutdown_watchdog_);
 921   base::TimeDelta actual_duration = duration;
 922
 923   chrome::VersionInfo::Channel channel = chrome::VersionInfo::GetChannel();
 924   if (channel == chrome::VersionInfo::CHANNEL_STABLE) {
 925     actual_duration *= 20;
 926   } else if (channel == chrome::VersionInfo::CHANNEL_BETA ||
 927              channel == chrome::VersionInfo::CHANNEL_DEV) {
 928     actual_duration *= 10;
 929   }
 930
 931 #if defined(OS_WIN)
 932   // On Windows XP, give twice the time for shutdown.
 933   if (base::win::GetVersion() <= base::win::VERSION_XP)
 934     actual_duration *= 2;
 935 #endif
 936
 937   shutdown_watchdog_ = new ShutdownWatchDogThread(actual_duration);
 938   shutdown_watchdog_->Arm();
 939 }