chrome/browser/metrics/thread_watcher.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/metrics/thread_watcher.h"
   6
   7 #include <math.h>  // ceil
   8
   9 #include "base/bind.h"
  10 #include "base/compiler_specific.h"
  11 #include "base/debug/dump_without_crashing.h"
  12 #include "base/lazy_instance.h"
  13 #include "base/location.h"
  14 #include "base/strings/string_number_conversions.h"
  15 #include "base/strings/string_split.h"
  16 #include "base/strings/string_tokenizer.h"
  17 #include "base/strings/stringprintf.h"
  18 #include "base/thread_task_runner_handle.h"
  19 #include "base/threading/thread_restrictions.h"
  20 #include "build/build_config.h"
  21 #include "chrome/browser/chrome_notification_types.h"
  22 #include "chrome/browser/metrics/thread_watcher_report_hang.h"
  23 #include "chrome/common/chrome_switches.h"
  24 #include "chrome/common/chrome_version_info.h"
  25 #include "chrome/common/logging_chrome.h"
  26 #include "content/public/browser/notification_service.h"
  27
  28 #if defined(OS_WIN)
  29 #include "base/win/windows_version.h"
  30 #endif
  31
  32 using content::BrowserThread;
  33
  34 // ThreadWatcher methods and members.
  35 ThreadWatcher::ThreadWatcher(const WatchingParams& params)
  36     : thread_id_(params.thread_id),
  37       thread_name_(params.thread_name),
  38       watched_runner_(
  39           BrowserThread::GetMessageLoopProxyForThread(params.thread_id)),
  40       sleep_time_(params.sleep_time),
  41       unresponsive_time_(params.unresponsive_time),
  42       ping_time_(base::TimeTicks::Now()),
  43       pong_time_(ping_time_),
  44       ping_sequence_number_(0),
  45       active_(false),
  46       ping_count_(params.unresponsive_threshold),
  47       response_time_histogram_(NULL),
  48       unresponsive_time_histogram_(NULL),
  49       unresponsive_count_(0),
  50       hung_processing_complete_(false),
  51       unresponsive_threshold_(params.unresponsive_threshold),
  52       crash_on_hang_(params.crash_on_hang),
  53       live_threads_threshold_(params.live_threads_threshold),
  54       weak_ptr_factory_(this) {
  55   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  56   Initialize();
  57 }
  58
  59 ThreadWatcher::~ThreadWatcher() {}
  60
  61 // static
  62 void ThreadWatcher::StartWatching(const WatchingParams& params) {
  63   DCHECK_GE(params.sleep_time.InMilliseconds(), 0);
  64   DCHECK_GE(params.unresponsive_time.InMilliseconds(),
  65             params.sleep_time.InMilliseconds());
  66
  67   // If we are not on WatchDogThread, then post a task to call StartWatching on
  68   // WatchDogThread.
  69   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
  70     WatchDogThread::PostTask(
  71         FROM_HERE,
  72         base::Bind(&ThreadWatcher::StartWatching, params));
  73     return;
  74   }
  75
  76   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  77
  78   // Create a new thread watcher object for the given thread and activate it.
  79   ThreadWatcher* watcher = new ThreadWatcher(params);
  80
  81   DCHECK(watcher);
  82   // If we couldn't register the thread watcher object, we are shutting down,
  83   // then don't activate thread watching.
  84   if (!ThreadWatcherList::IsRegistered(params.thread_id))
  85     return;
  86   watcher->ActivateThreadWatching();
  87 }
  88
  89 void ThreadWatcher::ActivateThreadWatching() {
  90   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  91   if (active_) return;
  92   active_ = true;
  93   ping_count_ = unresponsive_threshold_;
  94   ResetHangCounters();
  95   base::ThreadTaskRunnerHandle::Get()->PostTask(
  96       FROM_HERE, base::Bind(&ThreadWatcher::PostPingMessage,
  97                             weak_ptr_factory_.GetWeakPtr()));
  98 }
  99
 100 void ThreadWatcher::DeActivateThreadWatching() {
 101   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 102   active_ = false;
 103   ping_count_ = 0;
 104   weak_ptr_factory_.InvalidateWeakPtrs();
 105 }
 106
 107 void ThreadWatcher::WakeUp() {
 108   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 109   // There is some user activity, PostPingMessage task of thread watcher if
 110   // needed.
 111   if (!active_) return;
 112
 113   // Throw away the previous |unresponsive_count_| and start over again. Just
 114   // before going to sleep, |unresponsive_count_| could be very close to
 115   // |unresponsive_threshold_| and when user becomes active,
 116   // |unresponsive_count_| can go over |unresponsive_threshold_| if there was no
 117   // response for ping messages. Reset |unresponsive_count_| to start measuring
 118   // the unresponsiveness of the threads when system becomes active.
 119   unresponsive_count_ = 0;
 120
 121   if (ping_count_ <= 0) {
 122     ping_count_ = unresponsive_threshold_;
 123     ResetHangCounters();
 124     PostPingMessage();
 125   } else {
 126     ping_count_ = unresponsive_threshold_;
 127   }
 128 }
 129
 130 void ThreadWatcher::PostPingMessage() {
 131   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 132   // If we have stopped watching or if the user is idle, then stop sending
 133   // ping messages.
 134   if (!active_ || ping_count_ <= 0)
 135     return;
 136
 137   // Save the current time when we have sent ping message.
 138   ping_time_ = base::TimeTicks::Now();
 139
 140   // Send a ping message to the watched thread. Callback will be called on
 141   // the WatchDogThread.
 142   base::Closure callback(
 143       base::Bind(&ThreadWatcher::OnPongMessage, weak_ptr_factory_.GetWeakPtr(),
 144                  ping_sequence_number_));
 145   if (watched_runner_->PostTask(
 146           FROM_HERE,
 147           base::Bind(&ThreadWatcher::OnPingMessage, thread_id_,
 148                      callback))) {
 149       // Post a task to check the responsiveness of watched thread.
 150       base::MessageLoop::current()->PostDelayedTask(
 151           FROM_HERE,
 152           base::Bind(&ThreadWatcher::OnCheckResponsiveness,
 153                      weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
 154           unresponsive_time_);
 155   } else {
 156     // Watched thread might have gone away, stop watching it.
 157     DeActivateThreadWatching();
 158   }
 159 }
 160
 161 void ThreadWatcher::OnPongMessage(uint64 ping_sequence_number) {
 162   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 163
 164   // Record watched thread's response time.
 165   base::TimeTicks now = base::TimeTicks::Now();
 166   base::TimeDelta response_time = now - ping_time_;
 167   response_time_histogram_->AddTime(response_time);
 168
 169   // Save the current time when we have got pong message.
 170   pong_time_ = now;
 171
 172   // Check if there are any extra pings in flight.
 173   DCHECK_EQ(ping_sequence_number_, ping_sequence_number);
 174   if (ping_sequence_number_ != ping_sequence_number)
 175     return;
 176
 177   // Increment sequence number for the next ping message to indicate watched
 178   // thread is responsive.
 179   ++ping_sequence_number_;
 180
 181   // If we have stopped watching or if the user is idle, then stop sending
 182   // ping messages.
 183   if (!active_ || --ping_count_ <= 0)
 184     return;
 185
 186   base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 187       FROM_HERE, base::Bind(&ThreadWatcher::PostPingMessage,
 188                             weak_ptr_factory_.GetWeakPtr()),
 189       sleep_time_);
 190 }
 191
 192 void ThreadWatcher::OnCheckResponsiveness(uint64 ping_sequence_number) {
 193   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 194   // If we have stopped watching then consider thread as responding.
 195   if (!active_) {
 196     responsive_ = true;
 197     return;
 198   }
 199   // If the latest ping_sequence_number_ is not same as the ping_sequence_number
 200   // that is passed in, then we can assume OnPongMessage was called.
 201   // OnPongMessage increments ping_sequence_number_.
 202   if (ping_sequence_number_ != ping_sequence_number) {
 203     // Reset unresponsive_count_ to zero because we got a response from the
 204     // watched thread.
 205     ResetHangCounters();
 206
 207     responsive_ = true;
 208     return;
 209   }
 210   // Record that we got no response from watched thread.
 211   GotNoResponse();
 212
 213   // Post a task to check the responsiveness of watched thread.
 214   base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 215       FROM_HERE,
 216       base::Bind(&ThreadWatcher::OnCheckResponsiveness,
 217                  weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
 218       unresponsive_time_);
 219   responsive_ = false;
 220 }
 221
 222 void ThreadWatcher::Initialize() {
 223   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 224   ThreadWatcherList::Register(this);
 225
 226   const std::string response_time_histogram_name =
 227       "ThreadWatcher.ResponseTime." + thread_name_;
 228   response_time_histogram_ = base::Histogram::FactoryTimeGet(
 229       response_time_histogram_name,
 230       base::TimeDelta::FromMilliseconds(1),
 231       base::TimeDelta::FromSeconds(100), 50,
 232       base::Histogram::kUmaTargetedHistogramFlag);
 233
 234   const std::string unresponsive_time_histogram_name =
 235       "ThreadWatcher.Unresponsive." + thread_name_;
 236   unresponsive_time_histogram_ = base::Histogram::FactoryTimeGet(
 237       unresponsive_time_histogram_name,
 238       base::TimeDelta::FromMilliseconds(1),
 239       base::TimeDelta::FromSeconds(100), 50,
 240       base::Histogram::kUmaTargetedHistogramFlag);
 241
 242   const std::string responsive_count_histogram_name =
 243       "ThreadWatcher.ResponsiveThreads." + thread_name_;
 244   responsive_count_histogram_ = base::LinearHistogram::FactoryGet(
 245       responsive_count_histogram_name, 1, 10, 11,
 246       base::Histogram::kUmaTargetedHistogramFlag);
 247
 248   const std::string unresponsive_count_histogram_name =
 249       "ThreadWatcher.UnresponsiveThreads." + thread_name_;
 250   unresponsive_count_histogram_ = base::LinearHistogram::FactoryGet(
 251       unresponsive_count_histogram_name, 1, 10, 11,
 252       base::Histogram::kUmaTargetedHistogramFlag);
 253 }
 254
 255 // static
 256 void ThreadWatcher::OnPingMessage(const BrowserThread::ID& thread_id,
 257                                   const base::Closure& callback_task) {
 258   // This method is called on watched thread.
 259   DCHECK(BrowserThread::CurrentlyOn(thread_id));
 260   WatchDogThread::PostTask(FROM_HERE, callback_task);
 261 }
 262
 263 void ThreadWatcher::ResetHangCounters() {
 264   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 265   unresponsive_count_ = 0;
 266   hung_processing_complete_ = false;
 267 }
 268
 269 void ThreadWatcher::GotNoResponse() {
 270   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 271
 272   ++unresponsive_count_;
 273   if (!IsVeryUnresponsive())
 274     return;
 275
 276   // Record total unresponsive_time since last pong message.
 277   base::TimeDelta unresponse_time = base::TimeTicks::Now() - pong_time_;
 278   unresponsive_time_histogram_->AddTime(unresponse_time);
 279
 280   // We have already collected stats for the non-responding watched thread.
 281   if (hung_processing_complete_)
 282     return;
 283
 284   // Record how other threads are responding.
 285   uint32 responding_thread_count = 0;
 286   uint32 unresponding_thread_count = 0;
 287   ThreadWatcherList::GetStatusOfThreads(&responding_thread_count,
 288                                         &unresponding_thread_count);
 289
 290   // Record how many watched threads are responding.
 291   responsive_count_histogram_->Add(responding_thread_count);
 292
 293   // Record how many watched threads are not responding.
 294   unresponsive_count_histogram_->Add(unresponding_thread_count);
 295
 296   // Crash the browser if the watched thread is to be crashed on hang and if the
 297   // number of other threads responding is less than or equal to
 298   // live_threads_threshold_ and at least one other thread is responding.
 299   if (crash_on_hang_ &&
 300       responding_thread_count > 0 &&
 301       responding_thread_count <= live_threads_threshold_) {
 302     static bool crashed_once = false;
 303     if (!crashed_once) {
 304       crashed_once = true;
 305       metrics::CrashBecauseThreadWasUnresponsive(thread_id_);
 306     }
 307   }
 308
 309   hung_processing_complete_ = true;
 310 }
 311
 312 bool ThreadWatcher::IsVeryUnresponsive() {
 313   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 314   return unresponsive_count_ >= unresponsive_threshold_;
 315 }
 316
 317 // ThreadWatcherList methods and members.
 318 //
 319 // static
 320 ThreadWatcherList* ThreadWatcherList::g_thread_watcher_list_ = NULL;
 321 // static
 322 bool ThreadWatcherList::g_stopped_ = false;
 323 // static
 324 const int ThreadWatcherList::kSleepSeconds = 1;
 325 // static
 326 const int ThreadWatcherList::kUnresponsiveSeconds = 2;
 327 // static
 328 const int ThreadWatcherList::kUnresponsiveCount = 9;
 329 // static
 330 const int ThreadWatcherList::kLiveThreadsThreshold = 2;
 331 // static, non-const for tests.
 332 int ThreadWatcherList::g_initialize_delay_seconds = 120;
 333
 334 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds(
 335     uint32 live_threads_threshold,
 336     uint32 unresponsive_threshold)
 337     : live_threads_threshold(live_threads_threshold),
 338       unresponsive_threshold(unresponsive_threshold) {
 339 }
 340
 341 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds()
 342     : live_threads_threshold(kLiveThreadsThreshold),
 343       unresponsive_threshold(kUnresponsiveCount) {
 344 }
 345
 346 // static
 347 void ThreadWatcherList::StartWatchingAll(
 348     const base::CommandLine& command_line) {
 349   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 350   uint32 unresponsive_threshold;
 351   CrashOnHangThreadMap crash_on_hang_threads;
 352   ParseCommandLine(command_line,
 353                    &unresponsive_threshold,
 354                    &crash_on_hang_threads);
 355
 356   ThreadWatcherObserver::SetupNotifications(
 357       base::TimeDelta::FromSeconds(kSleepSeconds * unresponsive_threshold));
 358
 359   WatchDogThread::PostTask(
 360       FROM_HERE,
 361       base::Bind(&ThreadWatcherList::SetStopped, false));
 362
 363   if (!WatchDogThread::PostDelayedTask(
 364           FROM_HERE,
 365           base::Bind(&ThreadWatcherList::InitializeAndStartWatching,
 366                      unresponsive_threshold,
 367                      crash_on_hang_threads),
 368           base::TimeDelta::FromSeconds(g_initialize_delay_seconds))) {
 369     // Disarm() the startup timebomb, if we couldn't post the task to start the
 370     // ThreadWatcher (becasue WatchDog thread is not running).
 371     StartupTimeBomb::DisarmStartupTimeBomb();
 372   }
 373 }
 374
 375 // static
 376 void ThreadWatcherList::StopWatchingAll() {
 377   // TODO(rtenneti): Enable ThreadWatcher.
 378   ThreadWatcherObserver::RemoveNotifications();
 379   DeleteAll();
 380 }
 381
 382 // static
 383 void ThreadWatcherList::Register(ThreadWatcher* watcher) {
 384   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 385   if (!g_thread_watcher_list_)
 386     return;
 387   DCHECK(!g_thread_watcher_list_->Find(watcher->thread_id()));
 388   g_thread_watcher_list_->registered_[watcher->thread_id()] = watcher;
 389 }
 390
 391 // static
 392 bool ThreadWatcherList::IsRegistered(const BrowserThread::ID thread_id) {
 393   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 394   return NULL != ThreadWatcherList::Find(thread_id);
 395 }
 396
 397 // static
 398 void ThreadWatcherList::GetStatusOfThreads(uint32* responding_thread_count,
 399                                            uint32* unresponding_thread_count) {
 400   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 401   *responding_thread_count = 0;
 402   *unresponding_thread_count = 0;
 403   if (!g_thread_watcher_list_)
 404     return;
 405
 406   for (RegistrationList::iterator it =
 407            g_thread_watcher_list_->registered_.begin();
 408        g_thread_watcher_list_->registered_.end() != it;
 409        ++it) {
 410     if (it->second->IsVeryUnresponsive())
 411       ++(*unresponding_thread_count);
 412     else
 413       ++(*responding_thread_count);
 414   }
 415 }
 416
 417 // static
 418 void ThreadWatcherList::WakeUpAll() {
 419   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 420   if (!g_thread_watcher_list_)
 421     return;
 422
 423   for (RegistrationList::iterator it =
 424            g_thread_watcher_list_->registered_.begin();
 425        g_thread_watcher_list_->registered_.end() != it;
 426        ++it)
 427     it->second->WakeUp();
 428 }
 429
 430 ThreadWatcherList::ThreadWatcherList() {
 431   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 432   CHECK(!g_thread_watcher_list_);
 433   g_thread_watcher_list_ = this;
 434 }
 435
 436 ThreadWatcherList::~ThreadWatcherList() {
 437   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 438   DCHECK(this == g_thread_watcher_list_);
 439   g_thread_watcher_list_ = NULL;
 440 }
 441
 442 // static
 443 void ThreadWatcherList::ParseCommandLine(
 444     const base::CommandLine& command_line,
 445     uint32* unresponsive_threshold,
 446     CrashOnHangThreadMap* crash_on_hang_threads) {
 447   // Initialize |unresponsive_threshold| to a default value.
 448   *unresponsive_threshold = kUnresponsiveCount;
 449
 450   // Increase the unresponsive_threshold on the Stable and Beta channels to
 451   // reduce the number of crashes due to ThreadWatcher.
 452   chrome::VersionInfo::Channel channel = chrome::VersionInfo::GetChannel();
 453   if (channel == chrome::VersionInfo::CHANNEL_STABLE) {
 454     *unresponsive_threshold *= 4;
 455   } else if (channel == chrome::VersionInfo::CHANNEL_BETA) {
 456     *unresponsive_threshold *= 2;
 457   }
 458
 459 #if defined(OS_WIN)
 460   // For Windows XP (old systems), double the unresponsive_threshold to give
 461   // the OS a chance to schedule UI/IO threads a time slice to respond with a
 462   // pong message (to get around limitations with the OS).
 463   if (base::win::GetVersion() <= base::win::VERSION_XP)
 464     *unresponsive_threshold *= 2;
 465 #endif
 466
 467   uint32 crash_seconds = *unresponsive_threshold * kUnresponsiveSeconds;
 468   std::string crash_on_hang_thread_names;
 469   if (command_line.HasSwitch(switches::kCrashOnHangThreads)) {
 470     crash_on_hang_thread_names =
 471         command_line.GetSwitchValueASCII(switches::kCrashOnHangThreads);
 472   } else if (channel != chrome::VersionInfo::CHANNEL_STABLE) {
 473     // Default to crashing the browser if UI or IO or FILE threads are not
 474     // responsive except in stable channel.
 475     crash_on_hang_thread_names = base::StringPrintf(
 476         "UI:%d:%d,IO:%d:%d,FILE:%d:%d",
 477         kLiveThreadsThreshold, crash_seconds,
 478         kLiveThreadsThreshold, crash_seconds,
 479         kLiveThreadsThreshold, crash_seconds * 5);
 480   }
 481
 482   ParseCommandLineCrashOnHangThreads(crash_on_hang_thread_names,
 483                                      kLiveThreadsThreshold,
 484                                      crash_seconds,
 485                                      crash_on_hang_threads);
 486 }
 487
 488 // static
 489 void ThreadWatcherList::ParseCommandLineCrashOnHangThreads(
 490     const std::string& crash_on_hang_thread_names,
 491     uint32 default_live_threads_threshold,
 492     uint32 default_crash_seconds,
 493     CrashOnHangThreadMap* crash_on_hang_threads) {
 494   base::StringTokenizer tokens(crash_on_hang_thread_names, ",");
 495   while (tokens.GetNext()) {
 496     std::vector<base::StringPiece> values = base::SplitStringPiece(
 497         tokens.token_piece(), ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
 498     std::string thread_name = values[0].as_string();
 499
 500     uint32 live_threads_threshold = default_live_threads_threshold;
 501     uint32 crash_seconds = default_crash_seconds;
 502     if (values.size() >= 2 &&
 503         (!base::StringToUint(values[1], &live_threads_threshold))) {
 504       continue;
 505     }
 506     if (values.size() >= 3 &&
 507         (!base::StringToUint(values[2], &crash_seconds))) {
 508       continue;
 509     }
 510     uint32 unresponsive_threshold = static_cast<uint32>(
 511         ceil(static_cast<float>(crash_seconds) / kUnresponsiveSeconds));
 512
 513     CrashDataThresholds crash_data(live_threads_threshold,
 514                                    unresponsive_threshold);
 515     // Use the last specifier.
 516     (*crash_on_hang_threads)[thread_name] = crash_data;
 517   }
 518 }
 519
 520 // static
 521 void ThreadWatcherList::InitializeAndStartWatching(
 522     uint32 unresponsive_threshold,
 523     const CrashOnHangThreadMap& crash_on_hang_threads) {
 524   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 525
 526   // Disarm the startup timebomb, even if stop has been called.
 527   BrowserThread::PostTask(
 528       BrowserThread::UI,
 529       FROM_HERE,
 530       base::Bind(&StartupTimeBomb::DisarmStartupTimeBomb));
 531
 532   // This method is deferred in relationship to its StopWatchingAll()
 533   // counterpart. If a previous initialization has already happened, or if
 534   // stop has been called, there's nothing left to do here.
 535   if (g_thread_watcher_list_ || g_stopped_)
 536     return;
 537
 538   ThreadWatcherList* thread_watcher_list = new ThreadWatcherList();
 539   CHECK(thread_watcher_list);
 540
 541   // TODO(rtenneti): Because we don't generate crash dumps for ThreadWatcher in
 542   // stable channel, disable ThreadWatcher in stable and unknown channels. We
 543   // will also not collect histogram data in these channels until
 544   // http://crbug.com/426203 is fixed.
 545   chrome::VersionInfo::Channel channel = chrome::VersionInfo::GetChannel();
 546   if (channel == chrome::VersionInfo::CHANNEL_STABLE ||
 547       channel == chrome::VersionInfo::CHANNEL_UNKNOWN) {
 548     return;
 549   }
 550
 551   const base::TimeDelta kSleepTime =
 552       base::TimeDelta::FromSeconds(kSleepSeconds);
 553   const base::TimeDelta kUnresponsiveTime =
 554       base::TimeDelta::FromSeconds(kUnresponsiveSeconds);
 555
 556   StartWatching(BrowserThread::UI, "UI", kSleepTime, kUnresponsiveTime,
 557                 unresponsive_threshold, crash_on_hang_threads);
 558   StartWatching(BrowserThread::IO, "IO", kSleepTime, kUnresponsiveTime,
 559                 unresponsive_threshold, crash_on_hang_threads);
 560   StartWatching(BrowserThread::DB, "DB", kSleepTime, kUnresponsiveTime,
 561                 unresponsive_threshold, crash_on_hang_threads);
 562   StartWatching(BrowserThread::FILE, "FILE", kSleepTime, kUnresponsiveTime,
 563                 unresponsive_threshold, crash_on_hang_threads);
 564   StartWatching(BrowserThread::CACHE, "CACHE", kSleepTime, kUnresponsiveTime,
 565                 unresponsive_threshold, crash_on_hang_threads);
 566 }
 567
 568 // static
 569 void ThreadWatcherList::StartWatching(
 570     const BrowserThread::ID& thread_id,
 571     const std::string& thread_name,
 572     const base::TimeDelta& sleep_time,
 573     const base::TimeDelta& unresponsive_time,
 574     uint32 unresponsive_threshold,
 575     const CrashOnHangThreadMap& crash_on_hang_threads) {
 576   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 577
 578   CrashOnHangThreadMap::const_iterator it =
 579       crash_on_hang_threads.find(thread_name);
 580   bool crash_on_hang = false;
 581   uint32 live_threads_threshold = 0;
 582   if (it != crash_on_hang_threads.end()) {
 583     crash_on_hang = true;
 584     live_threads_threshold = it->second.live_threads_threshold;
 585     unresponsive_threshold = it->second.unresponsive_threshold;
 586   }
 587
 588   ThreadWatcher::StartWatching(
 589       ThreadWatcher::WatchingParams(thread_id,
 590                                     thread_name,
 591                                     sleep_time,
 592                                     unresponsive_time,
 593                                     unresponsive_threshold,
 594                                     crash_on_hang,
 595                                     live_threads_threshold));
 596 }
 597
 598 // static
 599 void ThreadWatcherList::DeleteAll() {
 600   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
 601     WatchDogThread::PostTask(
 602         FROM_HERE,
 603         base::Bind(&ThreadWatcherList::DeleteAll));
 604     return;
 605   }
 606
 607   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 608
 609   SetStopped(true);
 610
 611   if (!g_thread_watcher_list_)
 612     return;
 613
 614   // Delete all thread watcher objects.
 615   while (!g_thread_watcher_list_->registered_.empty()) {
 616     RegistrationList::iterator it = g_thread_watcher_list_->registered_.begin();
 617     delete it->second;
 618     g_thread_watcher_list_->registered_.erase(it);
 619   }
 620
 621   delete g_thread_watcher_list_;
 622 }
 623
 624 // static
 625 ThreadWatcher* ThreadWatcherList::Find(const BrowserThread::ID& thread_id) {
 626   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 627   if (!g_thread_watcher_list_)
 628     return NULL;
 629   RegistrationList::iterator it =
 630       g_thread_watcher_list_->registered_.find(thread_id);
 631   if (g_thread_watcher_list_->registered_.end() == it)
 632     return NULL;
 633   return it->second;
 634 }
 635
 636 // static
 637 void ThreadWatcherList::SetStopped(bool stopped) {
 638   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 639   g_stopped_ = stopped;
 640 }
 641
 642 // ThreadWatcherObserver methods and members.
 643 //
 644 // static
 645 ThreadWatcherObserver* ThreadWatcherObserver::g_thread_watcher_observer_ = NULL;
 646
 647 ThreadWatcherObserver::ThreadWatcherObserver(
 648     const base::TimeDelta& wakeup_interval)
 649     : last_wakeup_time_(base::TimeTicks::Now()),
 650       wakeup_interval_(wakeup_interval) {
 651   CHECK(!g_thread_watcher_observer_);
 652   g_thread_watcher_observer_ = this;
 653 }
 654
 655 ThreadWatcherObserver::~ThreadWatcherObserver() {
 656   DCHECK(this == g_thread_watcher_observer_);
 657   g_thread_watcher_observer_ = NULL;
 658 }
 659
 660 // static
 661 void ThreadWatcherObserver::SetupNotifications(
 662     const base::TimeDelta& wakeup_interval) {
 663   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 664   ThreadWatcherObserver* observer = new ThreadWatcherObserver(wakeup_interval);
 665   observer->registrar_.Add(
 666       observer,
 667       chrome::NOTIFICATION_BROWSER_OPENED,
 668       content::NotificationService::AllBrowserContextsAndSources());
 669   observer->registrar_.Add(observer,
 670                            chrome::NOTIFICATION_BROWSER_CLOSED,
 671                            content::NotificationService::AllSources());
 672   observer->registrar_.Add(observer,
 673                            chrome::NOTIFICATION_TAB_PARENTED,
 674                            content::NotificationService::AllSources());
 675   observer->registrar_.Add(observer,
 676                            chrome::NOTIFICATION_TAB_CLOSING,
 677                            content::NotificationService::AllSources());
 678   observer->registrar_.Add(observer,
 679                            content::NOTIFICATION_LOAD_START,
 680                            content::NotificationService::AllSources());
 681   observer->registrar_.Add(observer,
 682                            content::NOTIFICATION_LOAD_STOP,
 683                            content::NotificationService::AllSources());
 684   observer->registrar_.Add(observer,
 685                            content::NOTIFICATION_RENDERER_PROCESS_CLOSED,
 686                            content::NotificationService::AllSources());
 687   observer->registrar_.Add(observer,
 688                            content::NOTIFICATION_RENDER_WIDGET_HOST_HANG,
 689                            content::NotificationService::AllSources());
 690   observer->registrar_.Add(observer,
 691                            chrome::NOTIFICATION_OMNIBOX_OPENED_URL,
 692                            content::NotificationService::AllSources());
 693 }
 694
 695 // static
 696 void ThreadWatcherObserver::RemoveNotifications() {
 697   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 698   if (!g_thread_watcher_observer_)
 699     return;
 700   g_thread_watcher_observer_->registrar_.RemoveAll();
 701   delete g_thread_watcher_observer_;
 702 }
 703
 704 void ThreadWatcherObserver::Observe(
 705     int type,
 706     const content::NotificationSource& source,
 707     const content::NotificationDetails& details) {
 708   // There is some user activity, see if thread watchers are to be awakened.
 709   base::TimeTicks now = base::TimeTicks::Now();
 710   if ((now - last_wakeup_time_) < wakeup_interval_)
 711     return;
 712   last_wakeup_time_ = now;
 713   WatchDogThread::PostTask(
 714       FROM_HERE,
 715       base::Bind(&ThreadWatcherList::WakeUpAll));
 716 }
 717
 718 // WatchDogThread methods and members.
 719
 720 // This lock protects g_watchdog_thread.
 721 static base::LazyInstance<base::Lock>::Leaky
 722     g_watchdog_lock = LAZY_INSTANCE_INITIALIZER;
 723
 724 // The singleton of this class.
 725 static WatchDogThread* g_watchdog_thread = NULL;
 726
 727 WatchDogThread::WatchDogThread() : Thread("BrowserWatchdog") {
 728 }
 729
 730 WatchDogThread::~WatchDogThread() {
 731   Stop();
 732 }
 733
 734 // static
 735 bool WatchDogThread::CurrentlyOnWatchDogThread() {
 736   base::AutoLock lock(g_watchdog_lock.Get());
 737   return g_watchdog_thread &&
 738       g_watchdog_thread->message_loop() == base::MessageLoop::current();
 739 }
 740
 741 // static
 742 bool WatchDogThread::PostTask(const tracked_objects::Location& from_here,
 743                               const base::Closure& task) {
 744   return PostTaskHelper(from_here, task, base::TimeDelta());
 745 }
 746
 747 // static
 748 bool WatchDogThread::PostDelayedTask(const tracked_objects::Location& from_here,
 749                                      const base::Closure& task,
 750                                      base::TimeDelta delay) {
 751   return PostTaskHelper(from_here, task, delay);
 752 }
 753
 754 // static
 755 bool WatchDogThread::PostTaskHelper(
 756     const tracked_objects::Location& from_here,
 757     const base::Closure& task,
 758     base::TimeDelta delay) {
 759   {
 760     base::AutoLock lock(g_watchdog_lock.Get());
 761
 762     base::MessageLoop* message_loop = g_watchdog_thread ?
 763         g_watchdog_thread->message_loop() : NULL;
 764     if (message_loop) {
 765       message_loop->task_runner()->PostDelayedTask(from_here, task, delay);
 766       return true;
 767     }
 768   }
 769
 770   return false;
 771 }
 772
 773 void WatchDogThread::Init() {
 774   // This thread shouldn't be allowed to perform any blocking disk I/O.
 775   base::ThreadRestrictions::SetIOAllowed(false);
 776
 777   base::AutoLock lock(g_watchdog_lock.Get());
 778   CHECK(!g_watchdog_thread);
 779   g_watchdog_thread = this;
 780 }
 781
 782 void WatchDogThread::CleanUp() {
 783   base::AutoLock lock(g_watchdog_lock.Get());
 784   g_watchdog_thread = NULL;
 785 }
 786
 787 namespace {
 788
 789 // StartupWatchDogThread methods and members.
 790 //
 791 // Class for detecting hangs during startup.
 792 class StartupWatchDogThread : public base::Watchdog {
 793  public:
 794   // Constructor specifies how long the StartupWatchDogThread will wait before
 795   // alarming.
 796   explicit StartupWatchDogThread(const base::TimeDelta& duration)
 797       : base::Watchdog(duration, "Startup watchdog thread", true) {
 798   }
 799
 800   // Alarm is called if the time expires after an Arm() without someone calling
 801   // Disarm(). When Alarm goes off, in release mode we get the crash dump
 802   // without crashing and in debug mode we break into the debugger.
 803   void Alarm() override {
 804 #if !defined(NDEBUG)
 805     metrics::StartupHang();
 806     return;
 807 #elif !defined(OS_ANDROID)
 808     WatchDogThread::PostTask(FROM_HERE, base::Bind(&metrics::StartupHang));
 809     return;
 810 #else
 811     // TODO(rtenneti): Enable crashing for Android.
 812 #endif  // OS_ANDROID
 813   }
 814
 815  private:
 816   DISALLOW_COPY_AND_ASSIGN(StartupWatchDogThread);
 817 };
 818
 819 // ShutdownWatchDogThread methods and members.
 820 //
 821 // Class for detecting hangs during shutdown.
 822 class ShutdownWatchDogThread : public base::Watchdog {
 823  public:
 824   // Constructor specifies how long the ShutdownWatchDogThread will wait before
 825   // alarming.
 826   explicit ShutdownWatchDogThread(const base::TimeDelta& duration)
 827       : base::Watchdog(duration, "Shutdown watchdog thread", true) {
 828   }
 829
 830   // Alarm is called if the time expires after an Arm() without someone calling
 831   // Disarm(). We crash the browser if this method is called.
 832   void Alarm() override { metrics::ShutdownHang(); }
 833
 834  private:
 835   DISALLOW_COPY_AND_ASSIGN(ShutdownWatchDogThread);
 836 };
 837 }  // namespace
 838
 839 // StartupTimeBomb methods and members.
 840 //
 841 // static
 842 StartupTimeBomb* StartupTimeBomb::g_startup_timebomb_ = NULL;
 843
 844 StartupTimeBomb::StartupTimeBomb()
 845     : startup_watchdog_(NULL),
 846       thread_id_(base::PlatformThread::CurrentId()) {
 847   CHECK(!g_startup_timebomb_);
 848   g_startup_timebomb_ = this;
 849 }
 850
 851 StartupTimeBomb::~StartupTimeBomb() {
 852   DCHECK(this == g_startup_timebomb_);
 853   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 854   if (startup_watchdog_)
 855     Disarm();
 856   g_startup_timebomb_ = NULL;
 857 }
 858
 859 void StartupTimeBomb::Arm(const base::TimeDelta& duration) {
 860   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 861   DCHECK(!startup_watchdog_);
 862   startup_watchdog_ = new StartupWatchDogThread(duration);
 863   startup_watchdog_->Arm();
 864   return;
 865 }
 866
 867 void StartupTimeBomb::Disarm() {
 868   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 869   if (startup_watchdog_) {
 870     startup_watchdog_->Disarm();
 871     startup_watchdog_->Cleanup();
 872     DeleteStartupWatchdog();
 873   }
 874 }
 875
 876 void StartupTimeBomb::DeleteStartupWatchdog() {
 877   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 878   if (startup_watchdog_->IsJoinable()) {
 879     // Allow the watchdog thread to shutdown on UI. Watchdog thread shutdowns
 880     // very fast.
 881     base::ThreadRestrictions::SetIOAllowed(true);
 882     delete startup_watchdog_;
 883     startup_watchdog_ = NULL;
 884     return;
 885   }
 886   base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 887       FROM_HERE, base::Bind(&StartupTimeBomb::DeleteStartupWatchdog,
 888                             base::Unretained(this)),
 889       base::TimeDelta::FromSeconds(10));
 890 }
 891
 892 // static
 893 void StartupTimeBomb::DisarmStartupTimeBomb() {
 894   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 895   if (g_startup_timebomb_)
 896     g_startup_timebomb_->Disarm();
 897 }
 898
 899 // ShutdownWatcherHelper methods and members.
 900 //
 901 // ShutdownWatcherHelper is a wrapper class for detecting hangs during
 902 // shutdown.
 903 ShutdownWatcherHelper::ShutdownWatcherHelper()
 904     : shutdown_watchdog_(NULL),
 905       thread_id_(base::PlatformThread::CurrentId()) {
 906 }
 907
 908 ShutdownWatcherHelper::~ShutdownWatcherHelper() {
 909   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 910   if (shutdown_watchdog_) {
 911     shutdown_watchdog_->Disarm();
 912     delete shutdown_watchdog_;
 913     shutdown_watchdog_ = NULL;
 914   }
 915 }
 916
 917 void ShutdownWatcherHelper::Arm(const base::TimeDelta& duration) {
 918   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 919   DCHECK(!shutdown_watchdog_);
 920   base::TimeDelta actual_duration = duration;
 921
 922   chrome::VersionInfo::Channel channel = chrome::VersionInfo::GetChannel();
 923   if (channel == chrome::VersionInfo::CHANNEL_STABLE) {
 924     actual_duration *= 20;
 925   } else if (channel == chrome::VersionInfo::CHANNEL_BETA ||
 926              channel == chrome::VersionInfo::CHANNEL_DEV) {
 927     actual_duration *= 10;
 928   }
 929
 930 #if defined(OS_WIN)
 931   // On Windows XP, give twice the time for shutdown.
 932   if (base::win::GetVersion() <= base::win::VERSION_XP)
 933     actual_duration *= 2;
 934 #endif
 935
 936   shutdown_watchdog_ = new ShutdownWatchDogThread(actual_duration);
 937   shutdown_watchdog_->Arm();
 938 }