chrome/browser/metrics/thread_watcher.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/metrics/thread_watcher.h"
   6
   7 #include <math.h>  // ceil
   8
   9 #include "base/bind.h"
  10 #include "base/compiler_specific.h"
  11 #include "base/debug/dump_without_crashing.h"
  12 #include "base/lazy_instance.h"
  13 #include "base/location.h"
  14 #include "base/strings/string_number_conversions.h"
  15 #include "base/strings/string_split.h"
  16 #include "base/strings/string_tokenizer.h"
  17 #include "base/strings/stringprintf.h"
  18 #include "base/thread_task_runner_handle.h"
  19 #include "base/threading/thread_restrictions.h"
  20 #include "build/build_config.h"
  21 #include "chrome/browser/chrome_notification_types.h"
  22 #include "chrome/browser/metrics/thread_watcher_report_hang.h"
  23 #include "chrome/common/channel_info.h"
  24 #include "chrome/common/chrome_switches.h"
  25 #include "chrome/common/logging_chrome.h"
  26 #include "components/version_info/version_info.h"
  27 #include "content/public/browser/notification_service.h"
  28
  29 #if defined(OS_WIN)
  30 #include "base/win/windows_version.h"
  31 #endif
  32
  33 using content::BrowserThread;
  34
  35 // ThreadWatcher methods and members.
  36 ThreadWatcher::ThreadWatcher(const WatchingParams& params)
  37     : thread_id_(params.thread_id),
  38       thread_name_(params.thread_name),
  39       watched_runner_(
  40           BrowserThread::GetMessageLoopProxyForThread(params.thread_id)),
  41       sleep_time_(params.sleep_time),
  42       unresponsive_time_(params.unresponsive_time),
  43       ping_time_(base::TimeTicks::Now()),
  44       pong_time_(ping_time_),
  45       ping_sequence_number_(0),
  46       active_(false),
  47       ping_count_(params.unresponsive_threshold),
  48       response_time_histogram_(NULL),
  49       unresponsive_time_histogram_(NULL),
  50       unresponsive_count_(0),
  51       hung_processing_complete_(false),
  52       unresponsive_threshold_(params.unresponsive_threshold),
  53       crash_on_hang_(params.crash_on_hang),
  54       live_threads_threshold_(params.live_threads_threshold),
  55       weak_ptr_factory_(this) {
  56   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  57   Initialize();
  58 }
  59
  60 ThreadWatcher::~ThreadWatcher() {}
  61
  62 // static
  63 void ThreadWatcher::StartWatching(const WatchingParams& params) {
  64   DCHECK_GE(params.sleep_time.InMilliseconds(), 0);
  65   DCHECK_GE(params.unresponsive_time.InMilliseconds(),
  66             params.sleep_time.InMilliseconds());
  67
  68   // If we are not on WatchDogThread, then post a task to call StartWatching on
  69   // WatchDogThread.
  70   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
  71     WatchDogThread::PostTask(
  72         FROM_HERE,
  73         base::Bind(&ThreadWatcher::StartWatching, params));
  74     return;
  75   }
  76
  77   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  78
  79   // Create a new thread watcher object for the given thread and activate it.
  80   ThreadWatcher* watcher = new ThreadWatcher(params);
  81
  82   DCHECK(watcher);
  83   // If we couldn't register the thread watcher object, we are shutting down,
  84   // then don't activate thread watching.
  85   if (!ThreadWatcherList::IsRegistered(params.thread_id))
  86     return;
  87   watcher->ActivateThreadWatching();
  88 }
  89
  90 void ThreadWatcher::ActivateThreadWatching() {
  91   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  92   if (active_) return;
  93   active_ = true;
  94   ping_count_ = unresponsive_threshold_;
  95   ResetHangCounters();
  96   base::ThreadTaskRunnerHandle::Get()->PostTask(
  97       FROM_HERE, base::Bind(&ThreadWatcher::PostPingMessage,
  98                             weak_ptr_factory_.GetWeakPtr()));
  99 }
 100
 101 void ThreadWatcher::DeActivateThreadWatching() {
 102   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 103   active_ = false;
 104   ping_count_ = 0;
 105   weak_ptr_factory_.InvalidateWeakPtrs();
 106 }
 107
 108 void ThreadWatcher::WakeUp() {
 109   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 110   // There is some user activity, PostPingMessage task of thread watcher if
 111   // needed.
 112   if (!active_) return;
 113
 114   // Throw away the previous |unresponsive_count_| and start over again. Just
 115   // before going to sleep, |unresponsive_count_| could be very close to
 116   // |unresponsive_threshold_| and when user becomes active,
 117   // |unresponsive_count_| can go over |unresponsive_threshold_| if there was no
 118   // response for ping messages. Reset |unresponsive_count_| to start measuring
 119   // the unresponsiveness of the threads when system becomes active.
 120   unresponsive_count_ = 0;
 121
 122   if (ping_count_ <= 0) {
 123     ping_count_ = unresponsive_threshold_;
 124     ResetHangCounters();
 125     PostPingMessage();
 126   } else {
 127     ping_count_ = unresponsive_threshold_;
 128   }
 129 }
 130
 131 void ThreadWatcher::PostPingMessage() {
 132   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 133   // If we have stopped watching or if the user is idle, then stop sending
 134   // ping messages.
 135   if (!active_ || ping_count_ <= 0)
 136     return;
 137
 138   // Save the current time when we have sent ping message.
 139   ping_time_ = base::TimeTicks::Now();
 140
 141   // Send a ping message to the watched thread. Callback will be called on
 142   // the WatchDogThread.
 143   base::Closure callback(
 144       base::Bind(&ThreadWatcher::OnPongMessage, weak_ptr_factory_.GetWeakPtr(),
 145                  ping_sequence_number_));
 146   if (watched_runner_->PostTask(
 147           FROM_HERE,
 148           base::Bind(&ThreadWatcher::OnPingMessage, thread_id_,
 149                      callback))) {
 150       // Post a task to check the responsiveness of watched thread.
 151       base::MessageLoop::current()->PostDelayedTask(
 152           FROM_HERE,
 153           base::Bind(&ThreadWatcher::OnCheckResponsiveness,
 154                      weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
 155           unresponsive_time_);
 156   } else {
 157     // Watched thread might have gone away, stop watching it.
 158     DeActivateThreadWatching();
 159   }
 160 }
 161
 162 void ThreadWatcher::OnPongMessage(uint64 ping_sequence_number) {
 163   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 164
 165   // Record watched thread's response time.
 166   base::TimeTicks now = base::TimeTicks::Now();
 167   base::TimeDelta response_time = now - ping_time_;
 168   response_time_histogram_->AddTime(response_time);
 169
 170   // Save the current time when we have got pong message.
 171   pong_time_ = now;
 172
 173   // Check if there are any extra pings in flight.
 174   DCHECK_EQ(ping_sequence_number_, ping_sequence_number);
 175   if (ping_sequence_number_ != ping_sequence_number)
 176     return;
 177
 178   // Increment sequence number for the next ping message to indicate watched
 179   // thread is responsive.
 180   ++ping_sequence_number_;
 181
 182   // If we have stopped watching or if the user is idle, then stop sending
 183   // ping messages.
 184   if (!active_ || --ping_count_ <= 0)
 185     return;
 186
 187   base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 188       FROM_HERE, base::Bind(&ThreadWatcher::PostPingMessage,
 189                             weak_ptr_factory_.GetWeakPtr()),
 190       sleep_time_);
 191 }
 192
 193 void ThreadWatcher::OnCheckResponsiveness(uint64 ping_sequence_number) {
 194   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 195   // If we have stopped watching then consider thread as responding.
 196   if (!active_) {
 197     responsive_ = true;
 198     return;
 199   }
 200   // If the latest ping_sequence_number_ is not same as the ping_sequence_number
 201   // that is passed in, then we can assume OnPongMessage was called.
 202   // OnPongMessage increments ping_sequence_number_.
 203   if (ping_sequence_number_ != ping_sequence_number) {
 204     // Reset unresponsive_count_ to zero because we got a response from the
 205     // watched thread.
 206     ResetHangCounters();
 207
 208     responsive_ = true;
 209     return;
 210   }
 211   // Record that we got no response from watched thread.
 212   GotNoResponse();
 213
 214   // Post a task to check the responsiveness of watched thread.
 215   base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 216       FROM_HERE,
 217       base::Bind(&ThreadWatcher::OnCheckResponsiveness,
 218                  weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
 219       unresponsive_time_);
 220   responsive_ = false;
 221 }
 222
 223 void ThreadWatcher::Initialize() {
 224   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 225   ThreadWatcherList::Register(this);
 226
 227   const std::string response_time_histogram_name =
 228       "ThreadWatcher.ResponseTime." + thread_name_;
 229   response_time_histogram_ = base::Histogram::FactoryTimeGet(
 230       response_time_histogram_name,
 231       base::TimeDelta::FromMilliseconds(1),
 232       base::TimeDelta::FromSeconds(100), 50,
 233       base::Histogram::kUmaTargetedHistogramFlag);
 234
 235   const std::string unresponsive_time_histogram_name =
 236       "ThreadWatcher.Unresponsive." + thread_name_;
 237   unresponsive_time_histogram_ = base::Histogram::FactoryTimeGet(
 238       unresponsive_time_histogram_name,
 239       base::TimeDelta::FromMilliseconds(1),
 240       base::TimeDelta::FromSeconds(100), 50,
 241       base::Histogram::kUmaTargetedHistogramFlag);
 242
 243   const std::string responsive_count_histogram_name =
 244       "ThreadWatcher.ResponsiveThreads." + thread_name_;
 245   responsive_count_histogram_ = base::LinearHistogram::FactoryGet(
 246       responsive_count_histogram_name, 1, 10, 11,
 247       base::Histogram::kUmaTargetedHistogramFlag);
 248
 249   const std::string unresponsive_count_histogram_name =
 250       "ThreadWatcher.UnresponsiveThreads." + thread_name_;
 251   unresponsive_count_histogram_ = base::LinearHistogram::FactoryGet(
 252       unresponsive_count_histogram_name, 1, 10, 11,
 253       base::Histogram::kUmaTargetedHistogramFlag);
 254 }
 255
 256 // static
 257 void ThreadWatcher::OnPingMessage(const BrowserThread::ID& thread_id,
 258                                   const base::Closure& callback_task) {
 259   // This method is called on watched thread.
 260   DCHECK(BrowserThread::CurrentlyOn(thread_id));
 261   WatchDogThread::PostTask(FROM_HERE, callback_task);
 262 }
 263
 264 void ThreadWatcher::ResetHangCounters() {
 265   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 266   unresponsive_count_ = 0;
 267   hung_processing_complete_ = false;
 268 }
 269
 270 void ThreadWatcher::GotNoResponse() {
 271   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 272
 273   ++unresponsive_count_;
 274   if (!IsVeryUnresponsive())
 275     return;
 276
 277   // Record total unresponsive_time since last pong message.
 278   base::TimeDelta unresponse_time = base::TimeTicks::Now() - pong_time_;
 279   unresponsive_time_histogram_->AddTime(unresponse_time);
 280
 281   // We have already collected stats for the non-responding watched thread.
 282   if (hung_processing_complete_)
 283     return;
 284
 285   // Record how other threads are responding.
 286   uint32 responding_thread_count = 0;
 287   uint32 unresponding_thread_count = 0;
 288   ThreadWatcherList::GetStatusOfThreads(&responding_thread_count,
 289                                         &unresponding_thread_count);
 290
 291   // Record how many watched threads are responding.
 292   responsive_count_histogram_->Add(responding_thread_count);
 293
 294   // Record how many watched threads are not responding.
 295   unresponsive_count_histogram_->Add(unresponding_thread_count);
 296
 297   // Crash the browser if the watched thread is to be crashed on hang and if the
 298   // number of other threads responding is less than or equal to
 299   // live_threads_threshold_ and at least one other thread is responding.
 300   if (crash_on_hang_ &&
 301       responding_thread_count > 0 &&
 302       responding_thread_count <= live_threads_threshold_) {
 303     static bool crashed_once = false;
 304     if (!crashed_once) {
 305       crashed_once = true;
 306       metrics::CrashBecauseThreadWasUnresponsive(thread_id_);
 307     }
 308   }
 309
 310   hung_processing_complete_ = true;
 311 }
 312
 313 bool ThreadWatcher::IsVeryUnresponsive() {
 314   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 315   return unresponsive_count_ >= unresponsive_threshold_;
 316 }
 317
 318 // ThreadWatcherList methods and members.
 319 //
 320 // static
 321 ThreadWatcherList* ThreadWatcherList::g_thread_watcher_list_ = NULL;
 322 // static
 323 bool ThreadWatcherList::g_stopped_ = false;
 324 // static
 325 const int ThreadWatcherList::kSleepSeconds = 1;
 326 // static
 327 const int ThreadWatcherList::kUnresponsiveSeconds = 2;
 328 // static
 329 const int ThreadWatcherList::kUnresponsiveCount = 9;
 330 // static
 331 const int ThreadWatcherList::kLiveThreadsThreshold = 2;
 332 // static, non-const for tests.
 333 int ThreadWatcherList::g_initialize_delay_seconds = 120;
 334
 335 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds(
 336     uint32 live_threads_threshold,
 337     uint32 unresponsive_threshold)
 338     : live_threads_threshold(live_threads_threshold),
 339       unresponsive_threshold(unresponsive_threshold) {
 340 }
 341
 342 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds()
 343     : live_threads_threshold(kLiveThreadsThreshold),
 344       unresponsive_threshold(kUnresponsiveCount) {
 345 }
 346
 347 // static
 348 void ThreadWatcherList::StartWatchingAll(
 349     const base::CommandLine& command_line) {
 350   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 351   uint32 unresponsive_threshold;
 352   CrashOnHangThreadMap crash_on_hang_threads;
 353   ParseCommandLine(command_line,
 354                    &unresponsive_threshold,
 355                    &crash_on_hang_threads);
 356
 357   ThreadWatcherObserver::SetupNotifications(
 358       base::TimeDelta::FromSeconds(kSleepSeconds * unresponsive_threshold));
 359
 360   WatchDogThread::PostTask(
 361       FROM_HERE,
 362       base::Bind(&ThreadWatcherList::SetStopped, false));
 363
 364   if (!WatchDogThread::PostDelayedTask(
 365           FROM_HERE,
 366           base::Bind(&ThreadWatcherList::InitializeAndStartWatching,
 367                      unresponsive_threshold,
 368                      crash_on_hang_threads),
 369           base::TimeDelta::FromSeconds(g_initialize_delay_seconds))) {
 370     // Disarm() the startup timebomb, if we couldn't post the task to start the
 371     // ThreadWatcher (becasue WatchDog thread is not running).
 372     StartupTimeBomb::DisarmStartupTimeBomb();
 373   }
 374 }
 375
 376 // static
 377 void ThreadWatcherList::StopWatchingAll() {
 378   // TODO(rtenneti): Enable ThreadWatcher.
 379   ThreadWatcherObserver::RemoveNotifications();
 380   DeleteAll();
 381 }
 382
 383 // static
 384 void ThreadWatcherList::Register(ThreadWatcher* watcher) {
 385   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 386   if (!g_thread_watcher_list_)
 387     return;
 388   DCHECK(!g_thread_watcher_list_->Find(watcher->thread_id()));
 389   g_thread_watcher_list_->registered_[watcher->thread_id()] = watcher;
 390 }
 391
 392 // static
 393 bool ThreadWatcherList::IsRegistered(const BrowserThread::ID thread_id) {
 394   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 395   return NULL != ThreadWatcherList::Find(thread_id);
 396 }
 397
 398 // static
 399 void ThreadWatcherList::GetStatusOfThreads(uint32* responding_thread_count,
 400                                            uint32* unresponding_thread_count) {
 401   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 402   *responding_thread_count = 0;
 403   *unresponding_thread_count = 0;
 404   if (!g_thread_watcher_list_)
 405     return;
 406
 407   for (RegistrationList::iterator it =
 408            g_thread_watcher_list_->registered_.begin();
 409        g_thread_watcher_list_->registered_.end() != it;
 410        ++it) {
 411     if (it->second->IsVeryUnresponsive())
 412       ++(*unresponding_thread_count);
 413     else
 414       ++(*responding_thread_count);
 415   }
 416 }
 417
 418 // static
 419 void ThreadWatcherList::WakeUpAll() {
 420   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 421   if (!g_thread_watcher_list_)
 422     return;
 423
 424   for (RegistrationList::iterator it =
 425            g_thread_watcher_list_->registered_.begin();
 426        g_thread_watcher_list_->registered_.end() != it;
 427        ++it)
 428     it->second->WakeUp();
 429 }
 430
 431 ThreadWatcherList::ThreadWatcherList() {
 432   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 433   CHECK(!g_thread_watcher_list_);
 434   g_thread_watcher_list_ = this;
 435 }
 436
 437 ThreadWatcherList::~ThreadWatcherList() {
 438   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 439   DCHECK(this == g_thread_watcher_list_);
 440   g_thread_watcher_list_ = NULL;
 441 }
 442
 443 // static
 444 void ThreadWatcherList::ParseCommandLine(
 445     const base::CommandLine& command_line,
 446     uint32* unresponsive_threshold,
 447     CrashOnHangThreadMap* crash_on_hang_threads) {
 448   // Initialize |unresponsive_threshold| to a default value.
 449   *unresponsive_threshold = kUnresponsiveCount;
 450
 451   // Increase the unresponsive_threshold on the Stable and Beta channels to
 452   // reduce the number of crashes due to ThreadWatcher.
 453   version_info::Channel channel = chrome::GetChannel();
 454   if (channel == version_info::Channel::STABLE) {
 455     *unresponsive_threshold *= 4;
 456   } else if (channel == version_info::Channel::BETA) {
 457     *unresponsive_threshold *= 2;
 458   }
 459
 460 #if defined(OS_WIN)
 461   // For Windows XP (old systems), double the unresponsive_threshold to give
 462   // the OS a chance to schedule UI/IO threads a time slice to respond with a
 463   // pong message (to get around limitations with the OS).
 464   if (base::win::GetVersion() <= base::win::VERSION_XP)
 465     *unresponsive_threshold *= 2;
 466 #endif
 467
 468   uint32 crash_seconds = *unresponsive_threshold * kUnresponsiveSeconds;
 469   std::string crash_on_hang_thread_names;
 470   if (command_line.HasSwitch(switches::kCrashOnHangThreads)) {
 471     crash_on_hang_thread_names =
 472         command_line.GetSwitchValueASCII(switches::kCrashOnHangThreads);
 473   } else if (channel != version_info::Channel::STABLE) {
 474     // Default to crashing the browser if UI or IO or FILE threads are not
 475     // responsive except in stable channel.
 476     crash_on_hang_thread_names = base::StringPrintf(
 477         "UI:%d:%d,IO:%d:%d,FILE:%d:%d",
 478         kLiveThreadsThreshold, crash_seconds,
 479         kLiveThreadsThreshold, crash_seconds,
 480         kLiveThreadsThreshold, crash_seconds * 5);
 481   }
 482
 483   ParseCommandLineCrashOnHangThreads(crash_on_hang_thread_names,
 484                                      kLiveThreadsThreshold,
 485                                      crash_seconds,
 486                                      crash_on_hang_threads);
 487 }
 488
 489 // static
 490 void ThreadWatcherList::ParseCommandLineCrashOnHangThreads(
 491     const std::string& crash_on_hang_thread_names,
 492     uint32 default_live_threads_threshold,
 493     uint32 default_crash_seconds,
 494     CrashOnHangThreadMap* crash_on_hang_threads) {
 495   base::StringTokenizer tokens(crash_on_hang_thread_names, ",");
 496   while (tokens.GetNext()) {
 497     std::vector<base::StringPiece> values = base::SplitStringPiece(
 498         tokens.token_piece(), ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
 499     std::string thread_name = values[0].as_string();
 500
 501     uint32 live_threads_threshold = default_live_threads_threshold;
 502     uint32 crash_seconds = default_crash_seconds;
 503     if (values.size() >= 2 &&
 504         (!base::StringToUint(values[1], &live_threads_threshold))) {
 505       continue;
 506     }
 507     if (values.size() >= 3 &&
 508         (!base::StringToUint(values[2], &crash_seconds))) {
 509       continue;
 510     }
 511     uint32 unresponsive_threshold = static_cast<uint32>(
 512         ceil(static_cast<float>(crash_seconds) / kUnresponsiveSeconds));
 513
 514     CrashDataThresholds crash_data(live_threads_threshold,
 515                                    unresponsive_threshold);
 516     // Use the last specifier.
 517     (*crash_on_hang_threads)[thread_name] = crash_data;
 518   }
 519 }
 520
 521 // static
 522 void ThreadWatcherList::InitializeAndStartWatching(
 523     uint32 unresponsive_threshold,
 524     const CrashOnHangThreadMap& crash_on_hang_threads) {
 525   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 526
 527   // Disarm the startup timebomb, even if stop has been called.
 528   BrowserThread::PostTask(
 529       BrowserThread::UI,
 530       FROM_HERE,
 531       base::Bind(&StartupTimeBomb::DisarmStartupTimeBomb));
 532
 533   // This method is deferred in relationship to its StopWatchingAll()
 534   // counterpart. If a previous initialization has already happened, or if
 535   // stop has been called, there's nothing left to do here.
 536   if (g_thread_watcher_list_ || g_stopped_)
 537     return;
 538
 539   ThreadWatcherList* thread_watcher_list = new ThreadWatcherList();
 540   CHECK(thread_watcher_list);
 541
 542   // TODO(rtenneti): Because we don't generate crash dumps for ThreadWatcher in
 543   // stable channel, disable ThreadWatcher in stable and unknown channels. We
 544   // will also not collect histogram data in these channels until
 545   // http://crbug.com/426203 is fixed.
 546   version_info::Channel channel = chrome::GetChannel();
 547   if (channel == version_info::Channel::STABLE ||
 548       channel == version_info::Channel::UNKNOWN) {
 549     return;
 550   }
 551
 552   const base::TimeDelta kSleepTime =
 553       base::TimeDelta::FromSeconds(kSleepSeconds);
 554   const base::TimeDelta kUnresponsiveTime =
 555       base::TimeDelta::FromSeconds(kUnresponsiveSeconds);
 556
 557   StartWatching(BrowserThread::UI, "UI", kSleepTime, kUnresponsiveTime,
 558                 unresponsive_threshold, crash_on_hang_threads);
 559   StartWatching(BrowserThread::IO, "IO", kSleepTime, kUnresponsiveTime,
 560                 unresponsive_threshold, crash_on_hang_threads);
 561   StartWatching(BrowserThread::DB, "DB", kSleepTime, kUnresponsiveTime,
 562                 unresponsive_threshold, crash_on_hang_threads);
 563   StartWatching(BrowserThread::FILE, "FILE", kSleepTime, kUnresponsiveTime,
 564                 unresponsive_threshold, crash_on_hang_threads);
 565   StartWatching(BrowserThread::CACHE, "CACHE", kSleepTime, kUnresponsiveTime,
 566                 unresponsive_threshold, crash_on_hang_threads);
 567 }
 568
 569 // static
 570 void ThreadWatcherList::StartWatching(
 571     const BrowserThread::ID& thread_id,
 572     const std::string& thread_name,
 573     const base::TimeDelta& sleep_time,
 574     const base::TimeDelta& unresponsive_time,
 575     uint32 unresponsive_threshold,
 576     const CrashOnHangThreadMap& crash_on_hang_threads) {
 577   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 578
 579   CrashOnHangThreadMap::const_iterator it =
 580       crash_on_hang_threads.find(thread_name);
 581   bool crash_on_hang = false;
 582   uint32 live_threads_threshold = 0;
 583   if (it != crash_on_hang_threads.end()) {
 584     crash_on_hang = true;
 585     live_threads_threshold = it->second.live_threads_threshold;
 586     unresponsive_threshold = it->second.unresponsive_threshold;
 587   }
 588
 589   ThreadWatcher::StartWatching(
 590       ThreadWatcher::WatchingParams(thread_id,
 591                                     thread_name,
 592                                     sleep_time,
 593                                     unresponsive_time,
 594                                     unresponsive_threshold,
 595                                     crash_on_hang,
 596                                     live_threads_threshold));
 597 }
 598
 599 // static
 600 void ThreadWatcherList::DeleteAll() {
 601   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
 602     WatchDogThread::PostTask(
 603         FROM_HERE,
 604         base::Bind(&ThreadWatcherList::DeleteAll));
 605     return;
 606   }
 607
 608   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 609
 610   SetStopped(true);
 611
 612   if (!g_thread_watcher_list_)
 613     return;
 614
 615   // Delete all thread watcher objects.
 616   while (!g_thread_watcher_list_->registered_.empty()) {
 617     RegistrationList::iterator it = g_thread_watcher_list_->registered_.begin();
 618     delete it->second;
 619     g_thread_watcher_list_->registered_.erase(it);
 620   }
 621
 622   delete g_thread_watcher_list_;
 623 }
 624
 625 // static
 626 ThreadWatcher* ThreadWatcherList::Find(const BrowserThread::ID& thread_id) {
 627   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 628   if (!g_thread_watcher_list_)
 629     return NULL;
 630   RegistrationList::iterator it =
 631       g_thread_watcher_list_->registered_.find(thread_id);
 632   if (g_thread_watcher_list_->registered_.end() == it)
 633     return NULL;
 634   return it->second;
 635 }
 636
 637 // static
 638 void ThreadWatcherList::SetStopped(bool stopped) {
 639   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 640   g_stopped_ = stopped;
 641 }
 642
 643 // ThreadWatcherObserver methods and members.
 644 //
 645 // static
 646 ThreadWatcherObserver* ThreadWatcherObserver::g_thread_watcher_observer_ = NULL;
 647
 648 ThreadWatcherObserver::ThreadWatcherObserver(
 649     const base::TimeDelta& wakeup_interval)
 650     : last_wakeup_time_(base::TimeTicks::Now()),
 651       wakeup_interval_(wakeup_interval) {
 652   CHECK(!g_thread_watcher_observer_);
 653   g_thread_watcher_observer_ = this;
 654 }
 655
 656 ThreadWatcherObserver::~ThreadWatcherObserver() {
 657   DCHECK(this == g_thread_watcher_observer_);
 658   g_thread_watcher_observer_ = NULL;
 659 }
 660
 661 // static
 662 void ThreadWatcherObserver::SetupNotifications(
 663     const base::TimeDelta& wakeup_interval) {
 664   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 665   ThreadWatcherObserver* observer = new ThreadWatcherObserver(wakeup_interval);
 666   observer->registrar_.Add(
 667       observer,
 668       chrome::NOTIFICATION_BROWSER_OPENED,
 669       content::NotificationService::AllBrowserContextsAndSources());
 670   observer->registrar_.Add(observer,
 671                            chrome::NOTIFICATION_BROWSER_CLOSED,
 672                            content::NotificationService::AllSources());
 673   observer->registrar_.Add(observer,
 674                            chrome::NOTIFICATION_TAB_PARENTED,
 675                            content::NotificationService::AllSources());
 676   observer->registrar_.Add(observer,
 677                            chrome::NOTIFICATION_TAB_CLOSING,
 678                            content::NotificationService::AllSources());
 679   observer->registrar_.Add(observer,
 680                            content::NOTIFICATION_LOAD_START,
 681                            content::NotificationService::AllSources());
 682   observer->registrar_.Add(observer,
 683                            content::NOTIFICATION_LOAD_STOP,
 684                            content::NotificationService::AllSources());
 685   observer->registrar_.Add(observer,
 686                            content::NOTIFICATION_RENDERER_PROCESS_CLOSED,
 687                            content::NotificationService::AllSources());
 688   observer->registrar_.Add(observer,
 689                            content::NOTIFICATION_RENDER_WIDGET_HOST_HANG,
 690                            content::NotificationService::AllSources());
 691   observer->registrar_.Add(observer,
 692                            chrome::NOTIFICATION_OMNIBOX_OPENED_URL,
 693                            content::NotificationService::AllSources());
 694 }
 695
 696 // static
 697 void ThreadWatcherObserver::RemoveNotifications() {
 698   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 699   if (!g_thread_watcher_observer_)
 700     return;
 701   g_thread_watcher_observer_->registrar_.RemoveAll();
 702   delete g_thread_watcher_observer_;
 703 }
 704
 705 void ThreadWatcherObserver::Observe(
 706     int type,
 707     const content::NotificationSource& source,
 708     const content::NotificationDetails& details) {
 709   // There is some user activity, see if thread watchers are to be awakened.
 710   base::TimeTicks now = base::TimeTicks::Now();
 711   if ((now - last_wakeup_time_) < wakeup_interval_)
 712     return;
 713   last_wakeup_time_ = now;
 714   WatchDogThread::PostTask(
 715       FROM_HERE,
 716       base::Bind(&ThreadWatcherList::WakeUpAll));
 717 }
 718
 719 // WatchDogThread methods and members.
 720
 721 // This lock protects g_watchdog_thread.
 722 static base::LazyInstance<base::Lock>::Leaky
 723     g_watchdog_lock = LAZY_INSTANCE_INITIALIZER;
 724
 725 // The singleton of this class.
 726 static WatchDogThread* g_watchdog_thread = NULL;
 727
 728 WatchDogThread::WatchDogThread() : Thread("BrowserWatchdog") {
 729 }
 730
 731 WatchDogThread::~WatchDogThread() {
 732   Stop();
 733 }
 734
 735 // static
 736 bool WatchDogThread::CurrentlyOnWatchDogThread() {
 737   base::AutoLock lock(g_watchdog_lock.Get());
 738   return g_watchdog_thread &&
 739       g_watchdog_thread->message_loop() == base::MessageLoop::current();
 740 }
 741
 742 // static
 743 bool WatchDogThread::PostTask(const tracked_objects::Location& from_here,
 744                               const base::Closure& task) {
 745   return PostTaskHelper(from_here, task, base::TimeDelta());
 746 }
 747
 748 // static
 749 bool WatchDogThread::PostDelayedTask(const tracked_objects::Location& from_here,
 750                                      const base::Closure& task,
 751                                      base::TimeDelta delay) {
 752   return PostTaskHelper(from_here, task, delay);
 753 }
 754
 755 // static
 756 bool WatchDogThread::PostTaskHelper(
 757     const tracked_objects::Location& from_here,
 758     const base::Closure& task,
 759     base::TimeDelta delay) {
 760   {
 761     base::AutoLock lock(g_watchdog_lock.Get());
 762
 763     base::MessageLoop* message_loop = g_watchdog_thread ?
 764         g_watchdog_thread->message_loop() : NULL;
 765     if (message_loop) {
 766       message_loop->task_runner()->PostDelayedTask(from_here, task, delay);
 767       return true;
 768     }
 769   }
 770
 771   return false;
 772 }
 773
 774 void WatchDogThread::Init() {
 775   // This thread shouldn't be allowed to perform any blocking disk I/O.
 776   base::ThreadRestrictions::SetIOAllowed(false);
 777
 778   base::AutoLock lock(g_watchdog_lock.Get());
 779   CHECK(!g_watchdog_thread);
 780   g_watchdog_thread = this;
 781 }
 782
 783 void WatchDogThread::CleanUp() {
 784   base::AutoLock lock(g_watchdog_lock.Get());
 785   g_watchdog_thread = NULL;
 786 }
 787
 788 namespace {
 789
 790 // StartupWatchDogThread methods and members.
 791 //
 792 // Class for detecting hangs during startup.
 793 class StartupWatchDogThread : public base::Watchdog {
 794  public:
 795   // Constructor specifies how long the StartupWatchDogThread will wait before
 796   // alarming.
 797   explicit StartupWatchDogThread(const base::TimeDelta& duration)
 798       : base::Watchdog(duration, "Startup watchdog thread", true) {
 799   }
 800
 801   // Alarm is called if the time expires after an Arm() without someone calling
 802   // Disarm(). When Alarm goes off, in release mode we get the crash dump
 803   // without crashing and in debug mode we break into the debugger.
 804   void Alarm() override {
 805 #if !defined(NDEBUG)
 806     metrics::StartupHang();
 807     return;
 808 #elif !defined(OS_ANDROID)
 809     WatchDogThread::PostTask(FROM_HERE, base::Bind(&metrics::StartupHang));
 810     return;
 811 #else
 812     // TODO(rtenneti): Enable crashing for Android.
 813 #endif  // OS_ANDROID
 814   }
 815
 816  private:
 817   DISALLOW_COPY_AND_ASSIGN(StartupWatchDogThread);
 818 };
 819
 820 // ShutdownWatchDogThread methods and members.
 821 //
 822 // Class for detecting hangs during shutdown.
 823 class ShutdownWatchDogThread : public base::Watchdog {
 824  public:
 825   // Constructor specifies how long the ShutdownWatchDogThread will wait before
 826   // alarming.
 827   explicit ShutdownWatchDogThread(const base::TimeDelta& duration)
 828       : base::Watchdog(duration, "Shutdown watchdog thread", true) {
 829   }
 830
 831   // Alarm is called if the time expires after an Arm() without someone calling
 832   // Disarm(). We crash the browser if this method is called.
 833   void Alarm() override { metrics::ShutdownHang(); }
 834
 835  private:
 836   DISALLOW_COPY_AND_ASSIGN(ShutdownWatchDogThread);
 837 };
 838 }  // namespace
 839
 840 // StartupTimeBomb methods and members.
 841 //
 842 // static
 843 StartupTimeBomb* StartupTimeBomb::g_startup_timebomb_ = NULL;
 844
 845 StartupTimeBomb::StartupTimeBomb()
 846     : startup_watchdog_(NULL),
 847       thread_id_(base::PlatformThread::CurrentId()) {
 848   CHECK(!g_startup_timebomb_);
 849   g_startup_timebomb_ = this;
 850 }
 851
 852 StartupTimeBomb::~StartupTimeBomb() {
 853   DCHECK(this == g_startup_timebomb_);
 854   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 855   if (startup_watchdog_)
 856     Disarm();
 857   g_startup_timebomb_ = NULL;
 858 }
 859
 860 void StartupTimeBomb::Arm(const base::TimeDelta& duration) {
 861   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 862   DCHECK(!startup_watchdog_);
 863   startup_watchdog_ = new StartupWatchDogThread(duration);
 864   startup_watchdog_->Arm();
 865   return;
 866 }
 867
 868 void StartupTimeBomb::Disarm() {
 869   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 870   if (startup_watchdog_) {
 871     startup_watchdog_->Disarm();
 872     startup_watchdog_->Cleanup();
 873     DeleteStartupWatchdog();
 874   }
 875 }
 876
 877 void StartupTimeBomb::DeleteStartupWatchdog() {
 878   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 879   if (startup_watchdog_->IsJoinable()) {
 880     // Allow the watchdog thread to shutdown on UI. Watchdog thread shutdowns
 881     // very fast.
 882     base::ThreadRestrictions::SetIOAllowed(true);
 883     delete startup_watchdog_;
 884     startup_watchdog_ = NULL;
 885     return;
 886   }
 887   base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 888       FROM_HERE, base::Bind(&StartupTimeBomb::DeleteStartupWatchdog,
 889                             base::Unretained(this)),
 890       base::TimeDelta::FromSeconds(10));
 891 }
 892
 893 // static
 894 void StartupTimeBomb::DisarmStartupTimeBomb() {
 895   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 896   if (g_startup_timebomb_)
 897     g_startup_timebomb_->Disarm();
 898 }
 899
 900 // ShutdownWatcherHelper methods and members.
 901 //
 902 // ShutdownWatcherHelper is a wrapper class for detecting hangs during
 903 // shutdown.
 904 ShutdownWatcherHelper::ShutdownWatcherHelper()
 905     : shutdown_watchdog_(NULL),
 906       thread_id_(base::PlatformThread::CurrentId()) {
 907 }
 908
 909 ShutdownWatcherHelper::~ShutdownWatcherHelper() {
 910   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 911   if (shutdown_watchdog_) {
 912     shutdown_watchdog_->Disarm();
 913     delete shutdown_watchdog_;
 914     shutdown_watchdog_ = NULL;
 915   }
 916 }
 917
 918 void ShutdownWatcherHelper::Arm(const base::TimeDelta& duration) {
 919   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 920   DCHECK(!shutdown_watchdog_);
 921   base::TimeDelta actual_duration = duration;
 922
 923   version_info::Channel channel = chrome::GetChannel();
 924   if (channel == version_info::Channel::STABLE) {
 925     actual_duration *= 20;
 926   } else if (channel == version_info::Channel::BETA ||
 927              channel == version_info::Channel::DEV) {
 928     actual_duration *= 10;
 929   }
 930
 931 #if defined(OS_WIN)
 932   // On Windows XP, give twice the time for shutdown.
 933   if (base::win::GetVersion() <= base::win::VERSION_XP)
 934     actual_duration *= 2;
 935 #endif
 936
 937   shutdown_watchdog_ = new ShutdownWatchDogThread(actual_duration);
 938   shutdown_watchdog_->Arm();
 939 }