chrome/browser/metrics/thread_watcher.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/metrics/thread_watcher.h"
   6
   7 #include <math.h>  // ceil
   8
   9 #include "base/bind.h"
  10 #include "base/compiler_specific.h"
  11 #include "base/debug/dump_without_crashing.h"
  12 #include "base/lazy_instance.h"
  13 #include "base/metrics/field_trial.h"
  14 #include "base/strings/string_number_conversions.h"
  15 #include "base/strings/string_split.h"
  16 #include "base/strings/string_tokenizer.h"
  17 #include "base/strings/stringprintf.h"
  18 #include "base/threading/thread_restrictions.h"
  19 #include "build/build_config.h"
  20 #include "chrome/browser/chrome_notification_types.h"
  21 #include "chrome/browser/metrics/thread_watcher_report_hang.h"
  22 #include "chrome/common/chrome_switches.h"
  23 #include "chrome/common/chrome_version_info.h"
  24 #include "chrome/common/logging_chrome.h"
  25 #include "content/public/browser/notification_service.h"
  26
  27 #if defined(OS_WIN)
  28 #include "base/win/windows_version.h"
  29 #endif
  30
  31 using content::BrowserThread;
  32
  33 // ThreadWatcher methods and members.
  34 ThreadWatcher::ThreadWatcher(const WatchingParams& params)
  35     : thread_id_(params.thread_id),
  36       thread_name_(params.thread_name),
  37       watched_loop_(
  38           BrowserThread::GetMessageLoopProxyForThread(params.thread_id)),
  39       sleep_time_(params.sleep_time),
  40       unresponsive_time_(params.unresponsive_time),
  41       ping_time_(base::TimeTicks::Now()),
  42       pong_time_(ping_time_),
  43       ping_sequence_number_(0),
  44       active_(false),
  45       ping_count_(params.unresponsive_threshold),
  46       response_time_histogram_(NULL),
  47       unresponsive_time_histogram_(NULL),
  48       unresponsive_count_(0),
  49       hung_processing_complete_(false),
  50       unresponsive_threshold_(params.unresponsive_threshold),
  51       crash_on_hang_(params.crash_on_hang),
  52       live_threads_threshold_(params.live_threads_threshold),
  53       weak_ptr_factory_(this) {
  54   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  55   Initialize();
  56 }
  57
  58 ThreadWatcher::~ThreadWatcher() {}
  59
  60 // static
  61 void ThreadWatcher::StartWatching(const WatchingParams& params) {
  62   DCHECK_GE(params.sleep_time.InMilliseconds(), 0);
  63   DCHECK_GE(params.unresponsive_time.InMilliseconds(),
  64             params.sleep_time.InMilliseconds());
  65
  66   // If we are not on WatchDogThread, then post a task to call StartWatching on
  67   // WatchDogThread.
  68   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
  69     WatchDogThread::PostTask(
  70         FROM_HERE,
  71         base::Bind(&ThreadWatcher::StartWatching, params));
  72     return;
  73   }
  74
  75   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  76
  77   // Create a new thread watcher object for the given thread and activate it.
  78   ThreadWatcher* watcher = new ThreadWatcher(params);
  79
  80   DCHECK(watcher);
  81   // If we couldn't register the thread watcher object, we are shutting down,
  82   // then don't activate thread watching.
  83   if (!ThreadWatcherList::IsRegistered(params.thread_id))
  84     return;
  85   watcher->ActivateThreadWatching();
  86 }
  87
  88 void ThreadWatcher::ActivateThreadWatching() {
  89   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  90   if (active_) return;
  91   active_ = true;
  92   ping_count_ = unresponsive_threshold_;
  93   ResetHangCounters();
  94   base::MessageLoop::current()->PostTask(
  95       FROM_HERE,
  96       base::Bind(&ThreadWatcher::PostPingMessage,
  97                  weak_ptr_factory_.GetWeakPtr()));
  98 }
  99
 100 void ThreadWatcher::DeActivateThreadWatching() {
 101   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 102   active_ = false;
 103   ping_count_ = 0;
 104   weak_ptr_factory_.InvalidateWeakPtrs();
 105 }
 106
 107 void ThreadWatcher::WakeUp() {
 108   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 109   // There is some user activity, PostPingMessage task of thread watcher if
 110   // needed.
 111   if (!active_) return;
 112
 113   // Throw away the previous |unresponsive_count_| and start over again. Just
 114   // before going to sleep, |unresponsive_count_| could be very close to
 115   // |unresponsive_threshold_| and when user becomes active,
 116   // |unresponsive_count_| can go over |unresponsive_threshold_| if there was no
 117   // response for ping messages. Reset |unresponsive_count_| to start measuring
 118   // the unresponsiveness of the threads when system becomes active.
 119   unresponsive_count_ = 0;
 120
 121   if (ping_count_ <= 0) {
 122     ping_count_ = unresponsive_threshold_;
 123     ResetHangCounters();
 124     PostPingMessage();
 125   } else {
 126     ping_count_ = unresponsive_threshold_;
 127   }
 128 }
 129
 130 void ThreadWatcher::PostPingMessage() {
 131   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 132   // If we have stopped watching or if the user is idle, then stop sending
 133   // ping messages.
 134   if (!active_ || ping_count_ <= 0)
 135     return;
 136
 137   // Save the current time when we have sent ping message.
 138   ping_time_ = base::TimeTicks::Now();
 139
 140   // Send a ping message to the watched thread. Callback will be called on
 141   // the WatchDogThread.
 142   base::Closure callback(
 143       base::Bind(&ThreadWatcher::OnPongMessage, weak_ptr_factory_.GetWeakPtr(),
 144                  ping_sequence_number_));
 145   if (watched_loop_->PostTask(
 146           FROM_HERE,
 147           base::Bind(&ThreadWatcher::OnPingMessage, thread_id_,
 148                      callback))) {
 149       // Post a task to check the responsiveness of watched thread.
 150       base::MessageLoop::current()->PostDelayedTask(
 151           FROM_HERE,
 152           base::Bind(&ThreadWatcher::OnCheckResponsiveness,
 153                      weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
 154           unresponsive_time_);
 155   } else {
 156     // Watched thread might have gone away, stop watching it.
 157     DeActivateThreadWatching();
 158   }
 159 }
 160
 161 void ThreadWatcher::OnPongMessage(uint64 ping_sequence_number) {
 162   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 163
 164   // Record watched thread's response time.
 165   base::TimeTicks now = base::TimeTicks::Now();
 166   base::TimeDelta response_time = now - ping_time_;
 167   response_time_histogram_->AddTime(response_time);
 168
 169   // Save the current time when we have got pong message.
 170   pong_time_ = now;
 171
 172   // Check if there are any extra pings in flight.
 173   DCHECK_EQ(ping_sequence_number_, ping_sequence_number);
 174   if (ping_sequence_number_ != ping_sequence_number)
 175     return;
 176
 177   // Increment sequence number for the next ping message to indicate watched
 178   // thread is responsive.
 179   ++ping_sequence_number_;
 180
 181   // If we have stopped watching or if the user is idle, then stop sending
 182   // ping messages.
 183   if (!active_ || --ping_count_ <= 0)
 184     return;
 185
 186   base::MessageLoop::current()->PostDelayedTask(
 187       FROM_HERE,
 188       base::Bind(&ThreadWatcher::PostPingMessage,
 189                  weak_ptr_factory_.GetWeakPtr()),
 190       sleep_time_);
 191 }
 192
 193 void ThreadWatcher::OnCheckResponsiveness(uint64 ping_sequence_number) {
 194   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 195   // If we have stopped watching then consider thread as responding.
 196   if (!active_) {
 197     responsive_ = true;
 198     return;
 199   }
 200   // If the latest ping_sequence_number_ is not same as the ping_sequence_number
 201   // that is passed in, then we can assume OnPongMessage was called.
 202   // OnPongMessage increments ping_sequence_number_.
 203   if (ping_sequence_number_ != ping_sequence_number) {
 204     // Reset unresponsive_count_ to zero because we got a response from the
 205     // watched thread.
 206     ResetHangCounters();
 207
 208     responsive_ = true;
 209     return;
 210   }
 211   // Record that we got no response from watched thread.
 212   GotNoResponse();
 213
 214   // Post a task to check the responsiveness of watched thread.
 215   base::MessageLoop::current()->PostDelayedTask(
 216       FROM_HERE,
 217       base::Bind(&ThreadWatcher::OnCheckResponsiveness,
 218                  weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
 219       unresponsive_time_);
 220   responsive_ = false;
 221 }
 222
 223 void ThreadWatcher::Initialize() {
 224   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 225   ThreadWatcherList::Register(this);
 226
 227   const std::string response_time_histogram_name =
 228       "ThreadWatcher.ResponseTime." + thread_name_;
 229   response_time_histogram_ = base::Histogram::FactoryTimeGet(
 230       response_time_histogram_name,
 231       base::TimeDelta::FromMilliseconds(1),
 232       base::TimeDelta::FromSeconds(100), 50,
 233       base::Histogram::kUmaTargetedHistogramFlag);
 234
 235   const std::string unresponsive_time_histogram_name =
 236       "ThreadWatcher.Unresponsive." + thread_name_;
 237   unresponsive_time_histogram_ = base::Histogram::FactoryTimeGet(
 238       unresponsive_time_histogram_name,
 239       base::TimeDelta::FromMilliseconds(1),
 240       base::TimeDelta::FromSeconds(100), 50,
 241       base::Histogram::kUmaTargetedHistogramFlag);
 242
 243   const std::string responsive_count_histogram_name =
 244       "ThreadWatcher.ResponsiveThreads." + thread_name_;
 245   responsive_count_histogram_ = base::LinearHistogram::FactoryGet(
 246       responsive_count_histogram_name, 1, 10, 11,
 247       base::Histogram::kUmaTargetedHistogramFlag);
 248
 249   const std::string unresponsive_count_histogram_name =
 250       "ThreadWatcher.UnresponsiveThreads." + thread_name_;
 251   unresponsive_count_histogram_ = base::LinearHistogram::FactoryGet(
 252       unresponsive_count_histogram_name, 1, 10, 11,
 253       base::Histogram::kUmaTargetedHistogramFlag);
 254 }
 255
 256 // static
 257 void ThreadWatcher::OnPingMessage(const BrowserThread::ID& thread_id,
 258                                   const base::Closure& callback_task) {
 259   // This method is called on watched thread.
 260   DCHECK(BrowserThread::CurrentlyOn(thread_id));
 261   WatchDogThread::PostTask(FROM_HERE, callback_task);
 262 }
 263
 264 void ThreadWatcher::ResetHangCounters() {
 265   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 266   unresponsive_count_ = 0;
 267   hung_processing_complete_ = false;
 268 }
 269
 270 void ThreadWatcher::GotNoResponse() {
 271   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 272
 273   ++unresponsive_count_;
 274   if (!IsVeryUnresponsive())
 275     return;
 276
 277   // Record total unresponsive_time since last pong message.
 278   base::TimeDelta unresponse_time = base::TimeTicks::Now() - pong_time_;
 279   unresponsive_time_histogram_->AddTime(unresponse_time);
 280
 281   // We have already collected stats for the non-responding watched thread.
 282   if (hung_processing_complete_)
 283     return;
 284
 285   // Record how other threads are responding.
 286   uint32 responding_thread_count = 0;
 287   uint32 unresponding_thread_count = 0;
 288   ThreadWatcherList::GetStatusOfThreads(&responding_thread_count,
 289                                         &unresponding_thread_count);
 290
 291   // Record how many watched threads are responding.
 292   responsive_count_histogram_->Add(responding_thread_count);
 293
 294   // Record how many watched threads are not responding.
 295   unresponsive_count_histogram_->Add(unresponding_thread_count);
 296
 297   // Crash the browser if the watched thread is to be crashed on hang and if the
 298   // number of other threads responding is less than or equal to
 299   // live_threads_threshold_ and at least one other thread is responding.
 300   if (crash_on_hang_ &&
 301       responding_thread_count > 0 &&
 302       responding_thread_count <= live_threads_threshold_) {
 303     static bool crashed_once = false;
 304     if (!crashed_once) {
 305       crashed_once = true;
 306       metrics::CrashBecauseThreadWasUnresponsive(thread_id_);
 307     }
 308   }
 309
 310   hung_processing_complete_ = true;
 311 }
 312
 313 bool ThreadWatcher::IsVeryUnresponsive() {
 314   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 315   return unresponsive_count_ >= unresponsive_threshold_;
 316 }
 317
 318 // ThreadWatcherList methods and members.
 319 //
 320 // static
 321 ThreadWatcherList* ThreadWatcherList::g_thread_watcher_list_ = NULL;
 322 // static
 323 bool ThreadWatcherList::g_stopped_ = false;
 324 // static
 325 const int ThreadWatcherList::kSleepSeconds = 1;
 326 // static
 327 const int ThreadWatcherList::kUnresponsiveSeconds = 2;
 328 // static
 329 const int ThreadWatcherList::kUnresponsiveCount = 9;
 330 // static
 331 const int ThreadWatcherList::kLiveThreadsThreshold = 2;
 332 // static, non-const for tests.
 333 int ThreadWatcherList::g_initialize_delay_seconds = 120;
 334
 335 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds(
 336     uint32 live_threads_threshold,
 337     uint32 unresponsive_threshold)
 338     : live_threads_threshold(live_threads_threshold),
 339       unresponsive_threshold(unresponsive_threshold) {
 340 }
 341
 342 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds()
 343     : live_threads_threshold(kLiveThreadsThreshold),
 344       unresponsive_threshold(kUnresponsiveCount) {
 345 }
 346
 347 // static
 348 void ThreadWatcherList::StartWatchingAll(
 349     const base::CommandLine& command_line) {
 350   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 351   uint32 unresponsive_threshold;
 352   CrashOnHangThreadMap crash_on_hang_threads;
 353   ParseCommandLine(command_line,
 354                    &unresponsive_threshold,
 355                    &crash_on_hang_threads);
 356
 357   ThreadWatcherObserver::SetupNotifications(
 358       base::TimeDelta::FromSeconds(kSleepSeconds * unresponsive_threshold));
 359
 360   WatchDogThread::PostTask(
 361       FROM_HERE,
 362       base::Bind(&ThreadWatcherList::SetStopped, false));
 363
 364   if (!WatchDogThread::PostDelayedTask(
 365           FROM_HERE,
 366           base::Bind(&ThreadWatcherList::InitializeAndStartWatching,
 367                      unresponsive_threshold,
 368                      crash_on_hang_threads),
 369           base::TimeDelta::FromSeconds(g_initialize_delay_seconds))) {
 370     // Disarm() the startup timebomb, if we couldn't post the task to start the
 371     // ThreadWatcher (becasue WatchDog thread is not running).
 372     StartupTimeBomb::DisarmStartupTimeBomb();
 373   }
 374 }
 375
 376 // static
 377 void ThreadWatcherList::StopWatchingAll() {
 378   // TODO(rtenneti): Enable ThreadWatcher.
 379   ThreadWatcherObserver::RemoveNotifications();
 380   DeleteAll();
 381 }
 382
 383 // static
 384 void ThreadWatcherList::Register(ThreadWatcher* watcher) {
 385   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 386   if (!g_thread_watcher_list_)
 387     return;
 388   DCHECK(!g_thread_watcher_list_->Find(watcher->thread_id()));
 389   g_thread_watcher_list_->registered_[watcher->thread_id()] = watcher;
 390 }
 391
 392 // static
 393 bool ThreadWatcherList::IsRegistered(const BrowserThread::ID thread_id) {
 394   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 395   return NULL != ThreadWatcherList::Find(thread_id);
 396 }
 397
 398 // static
 399 void ThreadWatcherList::GetStatusOfThreads(uint32* responding_thread_count,
 400                                            uint32* unresponding_thread_count) {
 401   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 402   *responding_thread_count = 0;
 403   *unresponding_thread_count = 0;
 404   if (!g_thread_watcher_list_)
 405     return;
 406
 407   for (RegistrationList::iterator it =
 408            g_thread_watcher_list_->registered_.begin();
 409        g_thread_watcher_list_->registered_.end() != it;
 410        ++it) {
 411     if (it->second->IsVeryUnresponsive())
 412       ++(*unresponding_thread_count);
 413     else
 414       ++(*responding_thread_count);
 415   }
 416 }
 417
 418 // static
 419 void ThreadWatcherList::WakeUpAll() {
 420   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 421   if (!g_thread_watcher_list_)
 422     return;
 423
 424   for (RegistrationList::iterator it =
 425            g_thread_watcher_list_->registered_.begin();
 426        g_thread_watcher_list_->registered_.end() != it;
 427        ++it)
 428     it->second->WakeUp();
 429 }
 430
 431 ThreadWatcherList::ThreadWatcherList() {
 432   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 433   CHECK(!g_thread_watcher_list_);
 434   g_thread_watcher_list_ = this;
 435 }
 436
 437 ThreadWatcherList::~ThreadWatcherList() {
 438   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 439   DCHECK(this == g_thread_watcher_list_);
 440   g_thread_watcher_list_ = NULL;
 441 }
 442
 443 // static
 444 void ThreadWatcherList::ParseCommandLine(
 445     const base::CommandLine& command_line,
 446     uint32* unresponsive_threshold,
 447     CrashOnHangThreadMap* crash_on_hang_threads) {
 448   // Initialize |unresponsive_threshold| to a default value.
 449   *unresponsive_threshold = kUnresponsiveCount;
 450
 451   // Increase the unresponsive_threshold on the Stable and Beta channels to
 452   // reduce the number of crashes due to ThreadWatcher.
 453   chrome::VersionInfo::Channel channel = chrome::VersionInfo::GetChannel();
 454   if (channel == chrome::VersionInfo::CHANNEL_STABLE) {
 455     *unresponsive_threshold *= 4;
 456   } else if (channel == chrome::VersionInfo::CHANNEL_BETA) {
 457     *unresponsive_threshold *= 2;
 458   }
 459
 460 #if defined(OS_WIN)
 461   // For Windows XP (old systems), double the unresponsive_threshold to give
 462   // the OS a chance to schedule UI/IO threads a time slice to respond with a
 463   // pong message (to get around limitations with the OS).
 464   if (base::win::GetVersion() <= base::win::VERSION_XP)
 465     *unresponsive_threshold *= 2;
 466 #endif
 467
 468   uint32 crash_seconds = *unresponsive_threshold * kUnresponsiveSeconds;
 469   std::string crash_on_hang_thread_names;
 470   bool has_command_line_overwrite = false;
 471   if (command_line.HasSwitch(switches::kCrashOnHangThreads)) {
 472     crash_on_hang_thread_names =
 473         command_line.GetSwitchValueASCII(switches::kCrashOnHangThreads);
 474     has_command_line_overwrite = true;
 475   } else if (channel != chrome::VersionInfo::CHANNEL_STABLE) {
 476     // Default to crashing the browser if UI or IO or FILE threads are not
 477     // responsive except in stable channel.
 478     crash_on_hang_thread_names = base::StringPrintf(
 479         "UI:%d:%d,IO:%d:%d,FILE:%d:%d",
 480         kLiveThreadsThreshold, crash_seconds,
 481         kLiveThreadsThreshold, crash_seconds,
 482         kLiveThreadsThreshold, crash_seconds * 5);
 483   }
 484
 485   ParseCommandLineCrashOnHangThreads(crash_on_hang_thread_names,
 486                                      kLiveThreadsThreshold,
 487                                      crash_seconds,
 488                                      crash_on_hang_threads);
 489
 490   if (channel != chrome::VersionInfo::CHANNEL_CANARY ||
 491       has_command_line_overwrite) {
 492     return;
 493   }
 494
 495   const char* kFieldTrialName = "ThreadWatcher";
 496
 497   // Nothing else to be done if the trial has already been set (i.e., when
 498   // StartWatchingAll() has been already called once).
 499   if (base::FieldTrialList::TrialExists(kFieldTrialName))
 500     return;
 501
 502   // Set up a field trial for 100% of the users to crash if either UI or IO
 503   // thread is not responsive for 30 seconds (or 15 pings).
 504   scoped_refptr<base::FieldTrial> field_trial(
 505       base::FieldTrialList::FactoryGetFieldTrial(
 506           kFieldTrialName, 100, "default_hung_threads",
 507           2014, 10, 30, base::FieldTrial::SESSION_RANDOMIZED, NULL));
 508   int hung_thread_group = field_trial->AppendGroup("hung_thread", 100);
 509   if (field_trial->group() == hung_thread_group) {
 510     for (CrashOnHangThreadMap::iterator it = crash_on_hang_threads->begin();
 511          crash_on_hang_threads->end() != it;
 512          ++it) {
 513       if (it->first == "FILE")
 514         continue;
 515       it->second.live_threads_threshold = INT_MAX;
 516       if (it->first == "UI") {
 517         // TODO(rtenneti): set unresponsive threshold to 120 seconds to catch
 518         // the worst UI hangs and for fewer crashes due to ThreadWatcher. Reduce
 519         // it to a more reasonable time ala IO thread.
 520         it->second.unresponsive_threshold = 60;
 521       } else {
 522         it->second.unresponsive_threshold = 15;
 523       }
 524     }
 525   }
 526 }
 527
 528 // static
 529 void ThreadWatcherList::ParseCommandLineCrashOnHangThreads(
 530     const std::string& crash_on_hang_thread_names,
 531     uint32 default_live_threads_threshold,
 532     uint32 default_crash_seconds,
 533     CrashOnHangThreadMap* crash_on_hang_threads) {
 534   base::StringTokenizer tokens(crash_on_hang_thread_names, ",");
 535   std::vector<std::string> values;
 536   while (tokens.GetNext()) {
 537     const std::string& token = tokens.token();
 538     base::SplitString(token, ':', &values);
 539     std::string thread_name = values[0];
 540
 541     uint32 live_threads_threshold = default_live_threads_threshold;
 542     uint32 crash_seconds = default_crash_seconds;
 543     if (values.size() >= 2 &&
 544         (!base::StringToUint(values[1], &live_threads_threshold))) {
 545       continue;
 546     }
 547     if (values.size() >= 3 &&
 548         (!base::StringToUint(values[2], &crash_seconds))) {
 549       continue;
 550     }
 551     uint32 unresponsive_threshold = static_cast<uint32>(
 552         ceil(static_cast<float>(crash_seconds) / kUnresponsiveSeconds));
 553
 554     CrashDataThresholds crash_data(live_threads_threshold,
 555                                    unresponsive_threshold);
 556     // Use the last specifier.
 557     (*crash_on_hang_threads)[thread_name] = crash_data;
 558   }
 559 }
 560
 561 // static
 562 void ThreadWatcherList::InitializeAndStartWatching(
 563     uint32 unresponsive_threshold,
 564     const CrashOnHangThreadMap& crash_on_hang_threads) {
 565   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 566
 567   // Disarm the startup timebomb, even if stop has been called.
 568   BrowserThread::PostTask(
 569       BrowserThread::UI,
 570       FROM_HERE,
 571       base::Bind(&StartupTimeBomb::DisarmStartupTimeBomb));
 572
 573   // This method is deferred in relationship to its StopWatchingAll()
 574   // counterpart. If a previous initialization has already happened, or if
 575   // stop has been called, there's nothing left to do here.
 576   if (g_thread_watcher_list_ || g_stopped_)
 577     return;
 578
 579   ThreadWatcherList* thread_watcher_list = new ThreadWatcherList();
 580   CHECK(thread_watcher_list);
 581
 582   const base::TimeDelta kSleepTime =
 583       base::TimeDelta::FromSeconds(kSleepSeconds);
 584   const base::TimeDelta kUnresponsiveTime =
 585       base::TimeDelta::FromSeconds(kUnresponsiveSeconds);
 586
 587   StartWatching(BrowserThread::UI, "UI", kSleepTime, kUnresponsiveTime,
 588                 unresponsive_threshold, crash_on_hang_threads);
 589   StartWatching(BrowserThread::IO, "IO", kSleepTime, kUnresponsiveTime,
 590                 unresponsive_threshold, crash_on_hang_threads);
 591   StartWatching(BrowserThread::DB, "DB", kSleepTime, kUnresponsiveTime,
 592                 unresponsive_threshold, crash_on_hang_threads);
 593   StartWatching(BrowserThread::FILE, "FILE", kSleepTime, kUnresponsiveTime,
 594                 unresponsive_threshold, crash_on_hang_threads);
 595   StartWatching(BrowserThread::CACHE, "CACHE", kSleepTime, kUnresponsiveTime,
 596                 unresponsive_threshold, crash_on_hang_threads);
 597 }
 598
 599 // static
 600 void ThreadWatcherList::StartWatching(
 601     const BrowserThread::ID& thread_id,
 602     const std::string& thread_name,
 603     const base::TimeDelta& sleep_time,
 604     const base::TimeDelta& unresponsive_time,
 605     uint32 unresponsive_threshold,
 606     const CrashOnHangThreadMap& crash_on_hang_threads) {
 607   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 608
 609   CrashOnHangThreadMap::const_iterator it =
 610       crash_on_hang_threads.find(thread_name);
 611   bool crash_on_hang = false;
 612   uint32 live_threads_threshold = 0;
 613   if (it != crash_on_hang_threads.end()) {
 614     crash_on_hang = true;
 615     live_threads_threshold = it->second.live_threads_threshold;
 616     unresponsive_threshold = it->second.unresponsive_threshold;
 617   }
 618
 619   ThreadWatcher::StartWatching(
 620       ThreadWatcher::WatchingParams(thread_id,
 621                                     thread_name,
 622                                     sleep_time,
 623                                     unresponsive_time,
 624                                     unresponsive_threshold,
 625                                     crash_on_hang,
 626                                     live_threads_threshold));
 627 }
 628
 629 // static
 630 void ThreadWatcherList::DeleteAll() {
 631   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
 632     WatchDogThread::PostTask(
 633         FROM_HERE,
 634         base::Bind(&ThreadWatcherList::DeleteAll));
 635     return;
 636   }
 637
 638   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 639
 640   SetStopped(true);
 641
 642   if (!g_thread_watcher_list_)
 643     return;
 644
 645   // Delete all thread watcher objects.
 646   while (!g_thread_watcher_list_->registered_.empty()) {
 647     RegistrationList::iterator it = g_thread_watcher_list_->registered_.begin();
 648     delete it->second;
 649     g_thread_watcher_list_->registered_.erase(it);
 650   }
 651
 652   delete g_thread_watcher_list_;
 653 }
 654
 655 // static
 656 ThreadWatcher* ThreadWatcherList::Find(const BrowserThread::ID& thread_id) {
 657   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 658   if (!g_thread_watcher_list_)
 659     return NULL;
 660   RegistrationList::iterator it =
 661       g_thread_watcher_list_->registered_.find(thread_id);
 662   if (g_thread_watcher_list_->registered_.end() == it)
 663     return NULL;
 664   return it->second;
 665 }
 666
 667 // static
 668 void ThreadWatcherList::SetStopped(bool stopped) {
 669   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 670   g_stopped_ = stopped;
 671 }
 672
 673 // ThreadWatcherObserver methods and members.
 674 //
 675 // static
 676 ThreadWatcherObserver* ThreadWatcherObserver::g_thread_watcher_observer_ = NULL;
 677
 678 ThreadWatcherObserver::ThreadWatcherObserver(
 679     const base::TimeDelta& wakeup_interval)
 680     : last_wakeup_time_(base::TimeTicks::Now()),
 681       wakeup_interval_(wakeup_interval) {
 682   CHECK(!g_thread_watcher_observer_);
 683   g_thread_watcher_observer_ = this;
 684 }
 685
 686 ThreadWatcherObserver::~ThreadWatcherObserver() {
 687   DCHECK(this == g_thread_watcher_observer_);
 688   g_thread_watcher_observer_ = NULL;
 689 }
 690
 691 // static
 692 void ThreadWatcherObserver::SetupNotifications(
 693     const base::TimeDelta& wakeup_interval) {
 694   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 695   ThreadWatcherObserver* observer = new ThreadWatcherObserver(wakeup_interval);
 696   observer->registrar_.Add(
 697       observer,
 698       chrome::NOTIFICATION_BROWSER_OPENED,
 699       content::NotificationService::AllBrowserContextsAndSources());
 700   observer->registrar_.Add(observer,
 701                            chrome::NOTIFICATION_BROWSER_CLOSED,
 702                            content::NotificationService::AllSources());
 703   observer->registrar_.Add(observer,
 704                            chrome::NOTIFICATION_TAB_PARENTED,
 705                            content::NotificationService::AllSources());
 706   observer->registrar_.Add(observer,
 707                            chrome::NOTIFICATION_TAB_CLOSING,
 708                            content::NotificationService::AllSources());
 709   observer->registrar_.Add(observer,
 710                            content::NOTIFICATION_LOAD_START,
 711                            content::NotificationService::AllSources());
 712   observer->registrar_.Add(observer,
 713                            content::NOTIFICATION_LOAD_STOP,
 714                            content::NotificationService::AllSources());
 715   observer->registrar_.Add(observer,
 716                            content::NOTIFICATION_RENDERER_PROCESS_CLOSED,
 717                            content::NotificationService::AllSources());
 718   observer->registrar_.Add(observer,
 719                            content::NOTIFICATION_RENDER_WIDGET_HOST_HANG,
 720                            content::NotificationService::AllSources());
 721   observer->registrar_.Add(observer,
 722                            chrome::NOTIFICATION_OMNIBOX_OPENED_URL,
 723                            content::NotificationService::AllSources());
 724 }
 725
 726 // static
 727 void ThreadWatcherObserver::RemoveNotifications() {
 728   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 729   if (!g_thread_watcher_observer_)
 730     return;
 731   g_thread_watcher_observer_->registrar_.RemoveAll();
 732   delete g_thread_watcher_observer_;
 733 }
 734
 735 void ThreadWatcherObserver::Observe(
 736     int type,
 737     const content::NotificationSource& source,
 738     const content::NotificationDetails& details) {
 739   // There is some user activity, see if thread watchers are to be awakened.
 740   base::TimeTicks now = base::TimeTicks::Now();
 741   if ((now - last_wakeup_time_) < wakeup_interval_)
 742     return;
 743   last_wakeup_time_ = now;
 744   WatchDogThread::PostTask(
 745       FROM_HERE,
 746       base::Bind(&ThreadWatcherList::WakeUpAll));
 747 }
 748
 749 // WatchDogThread methods and members.
 750
 751 // This lock protects g_watchdog_thread.
 752 static base::LazyInstance<base::Lock>::Leaky
 753     g_watchdog_lock = LAZY_INSTANCE_INITIALIZER;
 754
 755 // The singleton of this class.
 756 static WatchDogThread* g_watchdog_thread = NULL;
 757
 758 WatchDogThread::WatchDogThread() : Thread("BrowserWatchdog") {
 759 }
 760
 761 WatchDogThread::~WatchDogThread() {
 762   Stop();
 763 }
 764
 765 // static
 766 bool WatchDogThread::CurrentlyOnWatchDogThread() {
 767   base::AutoLock lock(g_watchdog_lock.Get());
 768   return g_watchdog_thread &&
 769       g_watchdog_thread->message_loop() == base::MessageLoop::current();
 770 }
 771
 772 // static
 773 bool WatchDogThread::PostTask(const tracked_objects::Location& from_here,
 774                               const base::Closure& task) {
 775   return PostTaskHelper(from_here, task, base::TimeDelta());
 776 }
 777
 778 // static
 779 bool WatchDogThread::PostDelayedTask(const tracked_objects::Location& from_here,
 780                                      const base::Closure& task,
 781                                      base::TimeDelta delay) {
 782   return PostTaskHelper(from_here, task, delay);
 783 }
 784
 785 // static
 786 bool WatchDogThread::PostTaskHelper(
 787     const tracked_objects::Location& from_here,
 788     const base::Closure& task,
 789     base::TimeDelta delay) {
 790   {
 791     base::AutoLock lock(g_watchdog_lock.Get());
 792
 793     base::MessageLoop* message_loop = g_watchdog_thread ?
 794         g_watchdog_thread->message_loop() : NULL;
 795     if (message_loop) {
 796       message_loop->PostDelayedTask(from_here, task, delay);
 797       return true;
 798     }
 799   }
 800
 801   return false;
 802 }
 803
 804 void WatchDogThread::Init() {
 805   // This thread shouldn't be allowed to perform any blocking disk I/O.
 806   base::ThreadRestrictions::SetIOAllowed(false);
 807
 808   base::AutoLock lock(g_watchdog_lock.Get());
 809   CHECK(!g_watchdog_thread);
 810   g_watchdog_thread = this;
 811 }
 812
 813 void WatchDogThread::CleanUp() {
 814   base::AutoLock lock(g_watchdog_lock.Get());
 815   g_watchdog_thread = NULL;
 816 }
 817
 818 namespace {
 819
 820 // StartupWatchDogThread methods and members.
 821 //
 822 // Class for detecting hangs during startup.
 823 class StartupWatchDogThread : public base::Watchdog {
 824  public:
 825   // Constructor specifies how long the StartupWatchDogThread will wait before
 826   // alarming.
 827   explicit StartupWatchDogThread(const base::TimeDelta& duration)
 828       : base::Watchdog(duration, "Startup watchdog thread", true) {
 829   }
 830
 831   // Alarm is called if the time expires after an Arm() without someone calling
 832   // Disarm(). When Alarm goes off, in release mode we get the crash dump
 833   // without crashing and in debug mode we break into the debugger.
 834   void Alarm() override {
 835 #if !defined(NDEBUG)
 836     metrics::StartupHang();
 837     return;
 838 #elif !defined(OS_ANDROID)
 839     WatchDogThread::PostTask(FROM_HERE, base::Bind(&metrics::StartupHang));
 840     return;
 841 #else
 842     // TODO(rtenneti): Enable crashing for Android.
 843 #endif  // OS_ANDROID
 844   }
 845
 846  private:
 847   DISALLOW_COPY_AND_ASSIGN(StartupWatchDogThread);
 848 };
 849
 850 // ShutdownWatchDogThread methods and members.
 851 //
 852 // Class for detecting hangs during shutdown.
 853 class ShutdownWatchDogThread : public base::Watchdog {
 854  public:
 855   // Constructor specifies how long the ShutdownWatchDogThread will wait before
 856   // alarming.
 857   explicit ShutdownWatchDogThread(const base::TimeDelta& duration)
 858       : base::Watchdog(duration, "Shutdown watchdog thread", true) {
 859   }
 860
 861   // Alarm is called if the time expires after an Arm() without someone calling
 862   // Disarm(). We crash the browser if this method is called.
 863   void Alarm() override { metrics::ShutdownHang(); }
 864
 865  private:
 866   DISALLOW_COPY_AND_ASSIGN(ShutdownWatchDogThread);
 867 };
 868 }  // namespace
 869
 870 // StartupTimeBomb methods and members.
 871 //
 872 // static
 873 StartupTimeBomb* StartupTimeBomb::g_startup_timebomb_ = NULL;
 874
 875 StartupTimeBomb::StartupTimeBomb()
 876     : startup_watchdog_(NULL),
 877       thread_id_(base::PlatformThread::CurrentId()) {
 878   CHECK(!g_startup_timebomb_);
 879   g_startup_timebomb_ = this;
 880 }
 881
 882 StartupTimeBomb::~StartupTimeBomb() {
 883   DCHECK(this == g_startup_timebomb_);
 884   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 885   if (startup_watchdog_)
 886     Disarm();
 887   g_startup_timebomb_ = NULL;
 888 }
 889
 890 void StartupTimeBomb::Arm(const base::TimeDelta& duration) {
 891   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 892   DCHECK(!startup_watchdog_);
 893   startup_watchdog_ = new StartupWatchDogThread(duration);
 894   startup_watchdog_->Arm();
 895   return;
 896 }
 897
 898 void StartupTimeBomb::Disarm() {
 899   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 900   if (startup_watchdog_) {
 901     startup_watchdog_->Disarm();
 902     startup_watchdog_->Cleanup();
 903     DeleteStartupWatchdog();
 904   }
 905 }
 906
 907 void StartupTimeBomb::DeleteStartupWatchdog() {
 908   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 909   if (startup_watchdog_->IsJoinable()) {
 910     // Allow the watchdog thread to shutdown on UI. Watchdog thread shutdowns
 911     // very fast.
 912     base::ThreadRestrictions::SetIOAllowed(true);
 913     delete startup_watchdog_;
 914     startup_watchdog_ = NULL;
 915     return;
 916   }
 917   base::MessageLoop::current()->PostDelayedTask(
 918       FROM_HERE,
 919       base::Bind(&StartupTimeBomb::DeleteStartupWatchdog,
 920                  base::Unretained(this)),
 921       base::TimeDelta::FromSeconds(10));
 922 }
 923
 924 // static
 925 void StartupTimeBomb::DisarmStartupTimeBomb() {
 926   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 927   if (g_startup_timebomb_)
 928     g_startup_timebomb_->Disarm();
 929 }
 930
 931 // ShutdownWatcherHelper methods and members.
 932 //
 933 // ShutdownWatcherHelper is a wrapper class for detecting hangs during
 934 // shutdown.
 935 ShutdownWatcherHelper::ShutdownWatcherHelper()
 936     : shutdown_watchdog_(NULL),
 937       thread_id_(base::PlatformThread::CurrentId()) {
 938 }
 939
 940 ShutdownWatcherHelper::~ShutdownWatcherHelper() {
 941   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 942   if (shutdown_watchdog_) {
 943     shutdown_watchdog_->Disarm();
 944     delete shutdown_watchdog_;
 945     shutdown_watchdog_ = NULL;
 946   }
 947 }
 948
 949 void ShutdownWatcherHelper::Arm(const base::TimeDelta& duration) {
 950   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 951   DCHECK(!shutdown_watchdog_);
 952   base::TimeDelta actual_duration = duration;
 953
 954   chrome::VersionInfo::Channel channel = chrome::VersionInfo::GetChannel();
 955   if (channel == chrome::VersionInfo::CHANNEL_STABLE) {
 956     actual_duration *= 20;
 957   } else if (channel == chrome::VersionInfo::CHANNEL_BETA ||
 958              channel == chrome::VersionInfo::CHANNEL_DEV) {
 959     actual_duration *= 10;
 960   }
 961
 962 #if defined(OS_WIN)
 963   // On Windows XP, give twice the time for shutdown.
 964   if (base::win::GetVersion() <= base::win::VERSION_XP)
 965     actual_duration *= 2;
 966 #endif
 967
 968   shutdown_watchdog_ = new ShutdownWatchDogThread(actual_duration);
 969   shutdown_watchdog_->Arm();
 970 }