chrome/browser/metrics/thread_watcher.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/metrics/thread_watcher.h"
   6
   7 #include <math.h>  // ceil
   8
   9 #include "base/bind.h"
  10 #include "base/compiler_specific.h"
  11 #include "base/debug/dump_without_crashing.h"
  12 #include "base/lazy_instance.h"
  13 #include "base/metrics/field_trial.h"
  14 #include "base/strings/string_number_conversions.h"
  15 #include "base/strings/string_split.h"
  16 #include "base/strings/string_tokenizer.h"
  17 #include "base/strings/stringprintf.h"
  18 #include "base/threading/thread_restrictions.h"
  19 #include "build/build_config.h"
  20 #include "chrome/browser/chrome_notification_types.h"
  21 #include "chrome/browser/metrics/thread_watcher_report_hang.h"
  22 #include "chrome/common/chrome_switches.h"
  23 #include "chrome/common/chrome_version_info.h"
  24 #include "chrome/common/logging_chrome.h"
  25 #include "content/public/browser/notification_service.h"
  26
  27 #if defined(OS_WIN)
  28 #include "base/win/windows_version.h"
  29 #endif
  30
  31 using content::BrowserThread;
  32
  33 // ThreadWatcher methods and members.
  34 ThreadWatcher::ThreadWatcher(const WatchingParams& params)
  35     : thread_id_(params.thread_id),
  36       thread_name_(params.thread_name),
  37       watched_loop_(
  38           BrowserThread::GetMessageLoopProxyForThread(params.thread_id)),
  39       sleep_time_(params.sleep_time),
  40       unresponsive_time_(params.unresponsive_time),
  41       ping_time_(base::TimeTicks::Now()),
  42       pong_time_(ping_time_),
  43       ping_sequence_number_(0),
  44       active_(false),
  45       ping_count_(params.unresponsive_threshold),
  46       response_time_histogram_(NULL),
  47       unresponsive_time_histogram_(NULL),
  48       unresponsive_count_(0),
  49       hung_processing_complete_(false),
  50       unresponsive_threshold_(params.unresponsive_threshold),
  51       crash_on_hang_(params.crash_on_hang),
  52       live_threads_threshold_(params.live_threads_threshold),
  53       weak_ptr_factory_(this) {
  54   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  55   Initialize();
  56 }
  57
  58 ThreadWatcher::~ThreadWatcher() {}
  59
  60 // static
  61 void ThreadWatcher::StartWatching(const WatchingParams& params) {
  62   DCHECK_GE(params.sleep_time.InMilliseconds(), 0);
  63   DCHECK_GE(params.unresponsive_time.InMilliseconds(),
  64             params.sleep_time.InMilliseconds());
  65
  66   // If we are not on WatchDogThread, then post a task to call StartWatching on
  67   // WatchDogThread.
  68   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
  69     WatchDogThread::PostTask(
  70         FROM_HERE,
  71         base::Bind(&ThreadWatcher::StartWatching, params));
  72     return;
  73   }
  74
  75   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  76
  77   // Create a new thread watcher object for the given thread and activate it.
  78   ThreadWatcher* watcher = new ThreadWatcher(params);
  79
  80   DCHECK(watcher);
  81   // If we couldn't register the thread watcher object, we are shutting down,
  82   // then don't activate thread watching.
  83   if (!ThreadWatcherList::IsRegistered(params.thread_id))
  84     return;
  85   watcher->ActivateThreadWatching();
  86 }
  87
  88 void ThreadWatcher::ActivateThreadWatching() {
  89   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  90   if (active_) return;
  91   active_ = true;
  92   ping_count_ = unresponsive_threshold_;
  93   ResetHangCounters();
  94   base::MessageLoop::current()->PostTask(
  95       FROM_HERE,
  96       base::Bind(&ThreadWatcher::PostPingMessage,
  97                  weak_ptr_factory_.GetWeakPtr()));
  98 }
  99
 100 void ThreadWatcher::DeActivateThreadWatching() {
 101   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 102   active_ = false;
 103   ping_count_ = 0;
 104   weak_ptr_factory_.InvalidateWeakPtrs();
 105 }
 106
 107 void ThreadWatcher::WakeUp() {
 108   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 109   // There is some user activity, PostPingMessage task of thread watcher if
 110   // needed.
 111   if (!active_) return;
 112
 113   // Throw away the previous |unresponsive_count_| and start over again. Just
 114   // before going to sleep, |unresponsive_count_| could be very close to
 115   // |unresponsive_threshold_| and when user becomes active,
 116   // |unresponsive_count_| can go over |unresponsive_threshold_| if there was no
 117   // response for ping messages. Reset |unresponsive_count_| to start measuring
 118   // the unresponsiveness of the threads when system becomes active.
 119   unresponsive_count_ = 0;
 120
 121   if (ping_count_ <= 0) {
 122     ping_count_ = unresponsive_threshold_;
 123     ResetHangCounters();
 124     PostPingMessage();
 125   } else {
 126     ping_count_ = unresponsive_threshold_;
 127   }
 128 }
 129
 130 void ThreadWatcher::PostPingMessage() {
 131   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 132   // If we have stopped watching or if the user is idle, then stop sending
 133   // ping messages.
 134   if (!active_ || ping_count_ <= 0)
 135     return;
 136
 137   // Save the current time when we have sent ping message.
 138   ping_time_ = base::TimeTicks::Now();
 139
 140   // Send a ping message to the watched thread. Callback will be called on
 141   // the WatchDogThread.
 142   base::Closure callback(
 143       base::Bind(&ThreadWatcher::OnPongMessage, weak_ptr_factory_.GetWeakPtr(),
 144                  ping_sequence_number_));
 145   if (watched_loop_->PostTask(
 146           FROM_HERE,
 147           base::Bind(&ThreadWatcher::OnPingMessage, thread_id_,
 148                      callback))) {
 149       // Post a task to check the responsiveness of watched thread.
 150       base::MessageLoop::current()->PostDelayedTask(
 151           FROM_HERE,
 152           base::Bind(&ThreadWatcher::OnCheckResponsiveness,
 153                      weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
 154           unresponsive_time_);
 155   } else {
 156     // Watched thread might have gone away, stop watching it.
 157     DeActivateThreadWatching();
 158   }
 159 }
 160
 161 void ThreadWatcher::OnPongMessage(uint64 ping_sequence_number) {
 162   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 163
 164   // Record watched thread's response time.
 165   base::TimeTicks now = base::TimeTicks::Now();
 166   base::TimeDelta response_time = now - ping_time_;
 167   response_time_histogram_->AddTime(response_time);
 168
 169   // Save the current time when we have got pong message.
 170   pong_time_ = now;
 171
 172   // Check if there are any extra pings in flight.
 173   DCHECK_EQ(ping_sequence_number_, ping_sequence_number);
 174   if (ping_sequence_number_ != ping_sequence_number)
 175     return;
 176
 177   // Increment sequence number for the next ping message to indicate watched
 178   // thread is responsive.
 179   ++ping_sequence_number_;
 180
 181   // If we have stopped watching or if the user is idle, then stop sending
 182   // ping messages.
 183   if (!active_ || --ping_count_ <= 0)
 184     return;
 185
 186   base::MessageLoop::current()->PostDelayedTask(
 187       FROM_HERE,
 188       base::Bind(&ThreadWatcher::PostPingMessage,
 189                  weak_ptr_factory_.GetWeakPtr()),
 190       sleep_time_);
 191 }
 192
 193 void ThreadWatcher::OnCheckResponsiveness(uint64 ping_sequence_number) {
 194   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 195   // If we have stopped watching then consider thread as responding.
 196   if (!active_) {
 197     responsive_ = true;
 198     return;
 199   }
 200   // If the latest ping_sequence_number_ is not same as the ping_sequence_number
 201   // that is passed in, then we can assume OnPongMessage was called.
 202   // OnPongMessage increments ping_sequence_number_.
 203   if (ping_sequence_number_ != ping_sequence_number) {
 204     // Reset unresponsive_count_ to zero because we got a response from the
 205     // watched thread.
 206     ResetHangCounters();
 207
 208     responsive_ = true;
 209     return;
 210   }
 211   // Record that we got no response from watched thread.
 212   GotNoResponse();
 213
 214   // Post a task to check the responsiveness of watched thread.
 215   base::MessageLoop::current()->PostDelayedTask(
 216       FROM_HERE,
 217       base::Bind(&ThreadWatcher::OnCheckResponsiveness,
 218                  weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
 219       unresponsive_time_);
 220   responsive_ = false;
 221 }
 222
 223 void ThreadWatcher::Initialize() {
 224   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 225   ThreadWatcherList::Register(this);
 226
 227   const std::string response_time_histogram_name =
 228       "ThreadWatcher.ResponseTime." + thread_name_;
 229   response_time_histogram_ = base::Histogram::FactoryTimeGet(
 230       response_time_histogram_name,
 231       base::TimeDelta::FromMilliseconds(1),
 232       base::TimeDelta::FromSeconds(100), 50,
 233       base::Histogram::kUmaTargetedHistogramFlag);
 234
 235   const std::string unresponsive_time_histogram_name =
 236       "ThreadWatcher.Unresponsive." + thread_name_;
 237   unresponsive_time_histogram_ = base::Histogram::FactoryTimeGet(
 238       unresponsive_time_histogram_name,
 239       base::TimeDelta::FromMilliseconds(1),
 240       base::TimeDelta::FromSeconds(100), 50,
 241       base::Histogram::kUmaTargetedHistogramFlag);
 242
 243   const std::string responsive_count_histogram_name =
 244       "ThreadWatcher.ResponsiveThreads." + thread_name_;
 245   responsive_count_histogram_ = base::LinearHistogram::FactoryGet(
 246       responsive_count_histogram_name, 1, 10, 11,
 247       base::Histogram::kUmaTargetedHistogramFlag);
 248
 249   const std::string unresponsive_count_histogram_name =
 250       "ThreadWatcher.UnresponsiveThreads." + thread_name_;
 251   unresponsive_count_histogram_ = base::LinearHistogram::FactoryGet(
 252       unresponsive_count_histogram_name, 1, 10, 11,
 253       base::Histogram::kUmaTargetedHistogramFlag);
 254 }
 255
 256 // static
 257 void ThreadWatcher::OnPingMessage(const BrowserThread::ID& thread_id,
 258                                   const base::Closure& callback_task) {
 259   // This method is called on watched thread.
 260   DCHECK(BrowserThread::CurrentlyOn(thread_id));
 261   WatchDogThread::PostTask(FROM_HERE, callback_task);
 262 }
 263
 264 void ThreadWatcher::ResetHangCounters() {
 265   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 266   unresponsive_count_ = 0;
 267   hung_processing_complete_ = false;
 268 }
 269
 270 void ThreadWatcher::GotNoResponse() {
 271   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 272
 273   ++unresponsive_count_;
 274   if (!IsVeryUnresponsive())
 275     return;
 276
 277   // Record total unresponsive_time since last pong message.
 278   base::TimeDelta unresponse_time = base::TimeTicks::Now() - pong_time_;
 279   unresponsive_time_histogram_->AddTime(unresponse_time);
 280
 281   // We have already collected stats for the non-responding watched thread.
 282   if (hung_processing_complete_)
 283     return;
 284
 285   // Record how other threads are responding.
 286   uint32 responding_thread_count = 0;
 287   uint32 unresponding_thread_count = 0;
 288   ThreadWatcherList::GetStatusOfThreads(&responding_thread_count,
 289                                         &unresponding_thread_count);
 290
 291   // Record how many watched threads are responding.
 292   responsive_count_histogram_->Add(responding_thread_count);
 293
 294   // Record how many watched threads are not responding.
 295   unresponsive_count_histogram_->Add(unresponding_thread_count);
 296
 297   // Crash the browser if the watched thread is to be crashed on hang and if the
 298   // number of other threads responding is less than or equal to
 299   // live_threads_threshold_ and at least one other thread is responding.
 300   if (crash_on_hang_ &&
 301       responding_thread_count > 0 &&
 302       responding_thread_count <= live_threads_threshold_) {
 303     static bool crashed_once = false;
 304     if (!crashed_once) {
 305       crashed_once = true;
 306       metrics::CrashBecauseThreadWasUnresponsive(thread_id_);
 307     }
 308   }
 309
 310   hung_processing_complete_ = true;
 311 }
 312
 313 bool ThreadWatcher::IsVeryUnresponsive() {
 314   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 315   return unresponsive_count_ >= unresponsive_threshold_;
 316 }
 317
 318 // ThreadWatcherList methods and members.
 319 //
 320 // static
 321 ThreadWatcherList* ThreadWatcherList::g_thread_watcher_list_ = NULL;
 322 // static
 323 bool ThreadWatcherList::g_stopped_ = false;
 324 // static
 325 const int ThreadWatcherList::kSleepSeconds = 1;
 326 // static
 327 const int ThreadWatcherList::kUnresponsiveSeconds = 2;
 328 // static
 329 const int ThreadWatcherList::kUnresponsiveCount = 9;
 330 // static
 331 const int ThreadWatcherList::kLiveThreadsThreshold = 2;
 332 // static, non-const for tests.
 333 int ThreadWatcherList::g_initialize_delay_seconds = 120;
 334
 335 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds(
 336     uint32 live_threads_threshold,
 337     uint32 unresponsive_threshold)
 338     : live_threads_threshold(live_threads_threshold),
 339       unresponsive_threshold(unresponsive_threshold) {
 340 }
 341
 342 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds()
 343     : live_threads_threshold(kLiveThreadsThreshold),
 344       unresponsive_threshold(kUnresponsiveCount) {
 345 }
 346
 347 // static
 348 void ThreadWatcherList::StartWatchingAll(const CommandLine& command_line) {
 349   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 350   uint32 unresponsive_threshold;
 351   CrashOnHangThreadMap crash_on_hang_threads;
 352   ParseCommandLine(command_line,
 353                    &unresponsive_threshold,
 354                    &crash_on_hang_threads);
 355
 356   ThreadWatcherObserver::SetupNotifications(
 357       base::TimeDelta::FromSeconds(kSleepSeconds * unresponsive_threshold));
 358
 359   WatchDogThread::PostTask(
 360       FROM_HERE,
 361       base::Bind(&ThreadWatcherList::SetStopped, false));
 362
 363   if (!WatchDogThread::PostDelayedTask(
 364           FROM_HERE,
 365           base::Bind(&ThreadWatcherList::InitializeAndStartWatching,
 366                      unresponsive_threshold,
 367                      crash_on_hang_threads),
 368           base::TimeDelta::FromSeconds(g_initialize_delay_seconds))) {
 369     // Disarm() the startup timebomb, if we couldn't post the task to start the
 370     // ThreadWatcher (becasue WatchDog thread is not running).
 371     StartupTimeBomb::DisarmStartupTimeBomb();
 372   }
 373 }
 374
 375 // static
 376 void ThreadWatcherList::StopWatchingAll() {
 377   // TODO(rtenneti): Enable ThreadWatcher.
 378   ThreadWatcherObserver::RemoveNotifications();
 379   DeleteAll();
 380 }
 381
 382 // static
 383 void ThreadWatcherList::Register(ThreadWatcher* watcher) {
 384   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 385   if (!g_thread_watcher_list_)
 386     return;
 387   DCHECK(!g_thread_watcher_list_->Find(watcher->thread_id()));
 388   g_thread_watcher_list_->registered_[watcher->thread_id()] = watcher;
 389 }
 390
 391 // static
 392 bool ThreadWatcherList::IsRegistered(const BrowserThread::ID thread_id) {
 393   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 394   return NULL != ThreadWatcherList::Find(thread_id);
 395 }
 396
 397 // static
 398 void ThreadWatcherList::GetStatusOfThreads(uint32* responding_thread_count,
 399                                            uint32* unresponding_thread_count) {
 400   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 401   *responding_thread_count = 0;
 402   *unresponding_thread_count = 0;
 403   if (!g_thread_watcher_list_)
 404     return;
 405
 406   for (RegistrationList::iterator it =
 407            g_thread_watcher_list_->registered_.begin();
 408        g_thread_watcher_list_->registered_.end() != it;
 409        ++it) {
 410     if (it->second->IsVeryUnresponsive())
 411       ++(*unresponding_thread_count);
 412     else
 413       ++(*responding_thread_count);
 414   }
 415 }
 416
 417 // static
 418 void ThreadWatcherList::WakeUpAll() {
 419   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 420   if (!g_thread_watcher_list_)
 421     return;
 422
 423   for (RegistrationList::iterator it =
 424            g_thread_watcher_list_->registered_.begin();
 425        g_thread_watcher_list_->registered_.end() != it;
 426        ++it)
 427     it->second->WakeUp();
 428 }
 429
 430 ThreadWatcherList::ThreadWatcherList() {
 431   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 432   CHECK(!g_thread_watcher_list_);
 433   g_thread_watcher_list_ = this;
 434 }
 435
 436 ThreadWatcherList::~ThreadWatcherList() {
 437   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 438   DCHECK(this == g_thread_watcher_list_);
 439   g_thread_watcher_list_ = NULL;
 440 }
 441
 442 // static
 443 void ThreadWatcherList::ParseCommandLine(
 444     const CommandLine& command_line,
 445     uint32* unresponsive_threshold,
 446     CrashOnHangThreadMap* crash_on_hang_threads) {
 447   // Initialize |unresponsive_threshold| to a default value.
 448   *unresponsive_threshold = kUnresponsiveCount;
 449
 450   // Increase the unresponsive_threshold on the Stable and Beta channels to
 451   // reduce the number of crashes due to ThreadWatcher.
 452   chrome::VersionInfo::Channel channel = chrome::VersionInfo::GetChannel();
 453   if (channel == chrome::VersionInfo::CHANNEL_STABLE) {
 454     *unresponsive_threshold *= 4;
 455   } else if (channel == chrome::VersionInfo::CHANNEL_BETA) {
 456     *unresponsive_threshold *= 2;
 457   }
 458
 459 #if defined(OS_WIN)
 460   // For Windows XP (old systems), double the unresponsive_threshold to give
 461   // the OS a chance to schedule UI/IO threads a time slice to respond with a
 462   // pong message (to get around limitations with the OS).
 463   if (base::win::GetVersion() <= base::win::VERSION_XP)
 464     *unresponsive_threshold *= 2;
 465 #endif
 466
 467   uint32 crash_seconds = *unresponsive_threshold * kUnresponsiveSeconds;
 468   std::string crash_on_hang_thread_names;
 469   bool has_command_line_overwrite = false;
 470   if (command_line.HasSwitch(switches::kCrashOnHangThreads)) {
 471     crash_on_hang_thread_names =
 472         command_line.GetSwitchValueASCII(switches::kCrashOnHangThreads);
 473     has_command_line_overwrite = true;
 474   } else if (channel != chrome::VersionInfo::CHANNEL_STABLE) {
 475     // Default to crashing the browser if UI or IO or FILE threads are not
 476     // responsive except in stable channel.
 477     crash_on_hang_thread_names = base::StringPrintf(
 478         "UI:%d:%d,IO:%d:%d,FILE:%d:%d",
 479         kLiveThreadsThreshold, crash_seconds,
 480         kLiveThreadsThreshold, crash_seconds,
 481         kLiveThreadsThreshold, crash_seconds * 5);
 482   }
 483
 484   ParseCommandLineCrashOnHangThreads(crash_on_hang_thread_names,
 485                                      kLiveThreadsThreshold,
 486                                      crash_seconds,
 487                                      crash_on_hang_threads);
 488
 489   if (channel != chrome::VersionInfo::CHANNEL_CANARY ||
 490       has_command_line_overwrite) {
 491     return;
 492   }
 493
 494   const char* kFieldTrialName = "ThreadWatcher";
 495
 496   // Nothing else to be done if the trial has already been set (i.e., when
 497   // StartWatchingAll() has been already called once).
 498   if (base::FieldTrialList::TrialExists(kFieldTrialName))
 499     return;
 500
 501   // Set up a field trial for 100% of the users to crash if either UI or IO
 502   // thread is not responsive for 30 seconds (or 15 pings).
 503   scoped_refptr<base::FieldTrial> field_trial(
 504       base::FieldTrialList::FactoryGetFieldTrial(
 505           kFieldTrialName, 100, "default_hung_threads",
 506           2014, 10, 30, base::FieldTrial::SESSION_RANDOMIZED, NULL));
 507   int hung_thread_group = field_trial->AppendGroup("hung_thread", 100);
 508   if (field_trial->group() == hung_thread_group) {
 509     for (CrashOnHangThreadMap::iterator it = crash_on_hang_threads->begin();
 510          crash_on_hang_threads->end() != it;
 511          ++it) {
 512       if (it->first == "FILE")
 513         continue;
 514       it->second.live_threads_threshold = INT_MAX;
 515       if (it->first == "UI") {
 516         // TODO(rtenneti): set unresponsive threshold to 120 seconds to catch
 517         // the worst UI hangs and for fewer crashes due to ThreadWatcher. Reduce
 518         // it to a more reasonable time ala IO thread.
 519         it->second.unresponsive_threshold = 60;
 520       } else {
 521         it->second.unresponsive_threshold = 15;
 522       }
 523     }
 524   }
 525 }
 526
 527 // static
 528 void ThreadWatcherList::ParseCommandLineCrashOnHangThreads(
 529     const std::string& crash_on_hang_thread_names,
 530     uint32 default_live_threads_threshold,
 531     uint32 default_crash_seconds,
 532     CrashOnHangThreadMap* crash_on_hang_threads) {
 533   base::StringTokenizer tokens(crash_on_hang_thread_names, ",");
 534   std::vector<std::string> values;
 535   while (tokens.GetNext()) {
 536     const std::string& token = tokens.token();
 537     base::SplitString(token, ':', &values);
 538     std::string thread_name = values[0];
 539
 540     uint32 live_threads_threshold = default_live_threads_threshold;
 541     uint32 crash_seconds = default_crash_seconds;
 542     if (values.size() >= 2 &&
 543         (!base::StringToUint(values[1], &live_threads_threshold))) {
 544       continue;
 545     }
 546     if (values.size() >= 3 &&
 547         (!base::StringToUint(values[2], &crash_seconds))) {
 548       continue;
 549     }
 550     uint32 unresponsive_threshold = static_cast<uint32>(
 551         ceil(static_cast<float>(crash_seconds) / kUnresponsiveSeconds));
 552
 553     CrashDataThresholds crash_data(live_threads_threshold,
 554                                    unresponsive_threshold);
 555     // Use the last specifier.
 556     (*crash_on_hang_threads)[thread_name] = crash_data;
 557   }
 558 }
 559
 560 // static
 561 void ThreadWatcherList::InitializeAndStartWatching(
 562     uint32 unresponsive_threshold,
 563     const CrashOnHangThreadMap& crash_on_hang_threads) {
 564   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 565
 566   // Disarm the startup timebomb, even if stop has been called.
 567   BrowserThread::PostTask(
 568       BrowserThread::UI,
 569       FROM_HERE,
 570       base::Bind(&StartupTimeBomb::DisarmStartupTimeBomb));
 571
 572   // This method is deferred in relationship to its StopWatchingAll()
 573   // counterpart. If a previous initialization has already happened, or if
 574   // stop has been called, there's nothing left to do here.
 575   if (g_thread_watcher_list_ || g_stopped_)
 576     return;
 577
 578   ThreadWatcherList* thread_watcher_list = new ThreadWatcherList();
 579   CHECK(thread_watcher_list);
 580
 581   const base::TimeDelta kSleepTime =
 582       base::TimeDelta::FromSeconds(kSleepSeconds);
 583   const base::TimeDelta kUnresponsiveTime =
 584       base::TimeDelta::FromSeconds(kUnresponsiveSeconds);
 585
 586   StartWatching(BrowserThread::UI, "UI", kSleepTime, kUnresponsiveTime,
 587                 unresponsive_threshold, crash_on_hang_threads);
 588   StartWatching(BrowserThread::IO, "IO", kSleepTime, kUnresponsiveTime,
 589                 unresponsive_threshold, crash_on_hang_threads);
 590   StartWatching(BrowserThread::DB, "DB", kSleepTime, kUnresponsiveTime,
 591                 unresponsive_threshold, crash_on_hang_threads);
 592   StartWatching(BrowserThread::FILE, "FILE", kSleepTime, kUnresponsiveTime,
 593                 unresponsive_threshold, crash_on_hang_threads);
 594   StartWatching(BrowserThread::CACHE, "CACHE", kSleepTime, kUnresponsiveTime,
 595                 unresponsive_threshold, crash_on_hang_threads);
 596 }
 597
 598 // static
 599 void ThreadWatcherList::StartWatching(
 600     const BrowserThread::ID& thread_id,
 601     const std::string& thread_name,
 602     const base::TimeDelta& sleep_time,
 603     const base::TimeDelta& unresponsive_time,
 604     uint32 unresponsive_threshold,
 605     const CrashOnHangThreadMap& crash_on_hang_threads) {
 606   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 607
 608   CrashOnHangThreadMap::const_iterator it =
 609       crash_on_hang_threads.find(thread_name);
 610   bool crash_on_hang = false;
 611   uint32 live_threads_threshold = 0;
 612   if (it != crash_on_hang_threads.end()) {
 613     crash_on_hang = true;
 614     live_threads_threshold = it->second.live_threads_threshold;
 615     unresponsive_threshold = it->second.unresponsive_threshold;
 616   }
 617
 618   ThreadWatcher::StartWatching(
 619       ThreadWatcher::WatchingParams(thread_id,
 620                                     thread_name,
 621                                     sleep_time,
 622                                     unresponsive_time,
 623                                     unresponsive_threshold,
 624                                     crash_on_hang,
 625                                     live_threads_threshold));
 626 }
 627
 628 // static
 629 void ThreadWatcherList::DeleteAll() {
 630   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
 631     WatchDogThread::PostTask(
 632         FROM_HERE,
 633         base::Bind(&ThreadWatcherList::DeleteAll));
 634     return;
 635   }
 636
 637   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 638
 639   SetStopped(true);
 640
 641   if (!g_thread_watcher_list_)
 642     return;
 643
 644   // Delete all thread watcher objects.
 645   while (!g_thread_watcher_list_->registered_.empty()) {
 646     RegistrationList::iterator it = g_thread_watcher_list_->registered_.begin();
 647     delete it->second;
 648     g_thread_watcher_list_->registered_.erase(it);
 649   }
 650
 651   delete g_thread_watcher_list_;
 652 }
 653
 654 // static
 655 ThreadWatcher* ThreadWatcherList::Find(const BrowserThread::ID& thread_id) {
 656   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 657   if (!g_thread_watcher_list_)
 658     return NULL;
 659   RegistrationList::iterator it =
 660       g_thread_watcher_list_->registered_.find(thread_id);
 661   if (g_thread_watcher_list_->registered_.end() == it)
 662     return NULL;
 663   return it->second;
 664 }
 665
 666 // static
 667 void ThreadWatcherList::SetStopped(bool stopped) {
 668   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 669   g_stopped_ = stopped;
 670 }
 671
 672 // ThreadWatcherObserver methods and members.
 673 //
 674 // static
 675 ThreadWatcherObserver* ThreadWatcherObserver::g_thread_watcher_observer_ = NULL;
 676
 677 ThreadWatcherObserver::ThreadWatcherObserver(
 678     const base::TimeDelta& wakeup_interval)
 679     : last_wakeup_time_(base::TimeTicks::Now()),
 680       wakeup_interval_(wakeup_interval) {
 681   CHECK(!g_thread_watcher_observer_);
 682   g_thread_watcher_observer_ = this;
 683 }
 684
 685 ThreadWatcherObserver::~ThreadWatcherObserver() {
 686   DCHECK(this == g_thread_watcher_observer_);
 687   g_thread_watcher_observer_ = NULL;
 688 }
 689
 690 // static
 691 void ThreadWatcherObserver::SetupNotifications(
 692     const base::TimeDelta& wakeup_interval) {
 693   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 694   ThreadWatcherObserver* observer = new ThreadWatcherObserver(wakeup_interval);
 695   observer->registrar_.Add(
 696       observer,
 697       chrome::NOTIFICATION_BROWSER_OPENED,
 698       content::NotificationService::AllBrowserContextsAndSources());
 699   observer->registrar_.Add(observer,
 700                            chrome::NOTIFICATION_BROWSER_CLOSED,
 701                            content::NotificationService::AllSources());
 702   observer->registrar_.Add(observer,
 703                            chrome::NOTIFICATION_TAB_PARENTED,
 704                            content::NotificationService::AllSources());
 705   observer->registrar_.Add(observer,
 706                            chrome::NOTIFICATION_TAB_CLOSING,
 707                            content::NotificationService::AllSources());
 708   observer->registrar_.Add(observer,
 709                            content::NOTIFICATION_LOAD_START,
 710                            content::NotificationService::AllSources());
 711   observer->registrar_.Add(observer,
 712                            content::NOTIFICATION_LOAD_STOP,
 713                            content::NotificationService::AllSources());
 714   observer->registrar_.Add(observer,
 715                            content::NOTIFICATION_RENDERER_PROCESS_CLOSED,
 716                            content::NotificationService::AllSources());
 717   observer->registrar_.Add(observer,
 718                            content::NOTIFICATION_RENDER_WIDGET_HOST_HANG,
 719                            content::NotificationService::AllSources());
 720   observer->registrar_.Add(observer,
 721                            chrome::NOTIFICATION_OMNIBOX_OPENED_URL,
 722                            content::NotificationService::AllSources());
 723 }
 724
 725 // static
 726 void ThreadWatcherObserver::RemoveNotifications() {
 727   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 728   if (!g_thread_watcher_observer_)
 729     return;
 730   g_thread_watcher_observer_->registrar_.RemoveAll();
 731   delete g_thread_watcher_observer_;
 732 }
 733
 734 void ThreadWatcherObserver::Observe(
 735     int type,
 736     const content::NotificationSource& source,
 737     const content::NotificationDetails& details) {
 738   // There is some user activity, see if thread watchers are to be awakened.
 739   base::TimeTicks now = base::TimeTicks::Now();
 740   if ((now - last_wakeup_time_) < wakeup_interval_)
 741     return;
 742   last_wakeup_time_ = now;
 743   WatchDogThread::PostTask(
 744       FROM_HERE,
 745       base::Bind(&ThreadWatcherList::WakeUpAll));
 746 }
 747
 748 // WatchDogThread methods and members.
 749
 750 // This lock protects g_watchdog_thread.
 751 static base::LazyInstance<base::Lock>::Leaky
 752     g_watchdog_lock = LAZY_INSTANCE_INITIALIZER;
 753
 754 // The singleton of this class.
 755 static WatchDogThread* g_watchdog_thread = NULL;
 756
 757 WatchDogThread::WatchDogThread() : Thread("BrowserWatchdog") {
 758 }
 759
 760 WatchDogThread::~WatchDogThread() {
 761   Stop();
 762 }
 763
 764 // static
 765 bool WatchDogThread::CurrentlyOnWatchDogThread() {
 766   base::AutoLock lock(g_watchdog_lock.Get());
 767   return g_watchdog_thread &&
 768       g_watchdog_thread->message_loop() == base::MessageLoop::current();
 769 }
 770
 771 // static
 772 bool WatchDogThread::PostTask(const tracked_objects::Location& from_here,
 773                               const base::Closure& task) {
 774   return PostTaskHelper(from_here, task, base::TimeDelta());
 775 }
 776
 777 // static
 778 bool WatchDogThread::PostDelayedTask(const tracked_objects::Location& from_here,
 779                                      const base::Closure& task,
 780                                      base::TimeDelta delay) {
 781   return PostTaskHelper(from_here, task, delay);
 782 }
 783
 784 // static
 785 bool WatchDogThread::PostTaskHelper(
 786     const tracked_objects::Location& from_here,
 787     const base::Closure& task,
 788     base::TimeDelta delay) {
 789   {
 790     base::AutoLock lock(g_watchdog_lock.Get());
 791
 792     base::MessageLoop* message_loop = g_watchdog_thread ?
 793         g_watchdog_thread->message_loop() : NULL;
 794     if (message_loop) {
 795       message_loop->PostDelayedTask(from_here, task, delay);
 796       return true;
 797     }
 798   }
 799
 800   return false;
 801 }
 802
 803 void WatchDogThread::Init() {
 804   // This thread shouldn't be allowed to perform any blocking disk I/O.
 805   base::ThreadRestrictions::SetIOAllowed(false);
 806
 807   base::AutoLock lock(g_watchdog_lock.Get());
 808   CHECK(!g_watchdog_thread);
 809   g_watchdog_thread = this;
 810 }
 811
 812 void WatchDogThread::CleanUp() {
 813   base::AutoLock lock(g_watchdog_lock.Get());
 814   g_watchdog_thread = NULL;
 815 }
 816
 817 namespace {
 818
 819 // StartupWatchDogThread methods and members.
 820 //
 821 // Class for detecting hangs during startup.
 822 class StartupWatchDogThread : public base::Watchdog {
 823  public:
 824   // Constructor specifies how long the StartupWatchDogThread will wait before
 825   // alarming.
 826   explicit StartupWatchDogThread(const base::TimeDelta& duration)
 827       : base::Watchdog(duration, "Startup watchdog thread", true) {
 828   }
 829
 830   // Alarm is called if the time expires after an Arm() without someone calling
 831   // Disarm(). When Alarm goes off, in release mode we get the crash dump
 832   // without crashing and in debug mode we break into the debugger.
 833   void Alarm() override {
 834 #if !defined(NDEBUG)
 835     metrics::StartupHang();
 836     return;
 837 #elif !defined(OS_ANDROID)
 838     WatchDogThread::PostTask(FROM_HERE, base::Bind(&metrics::StartupHang));
 839     return;
 840 #else
 841     // TODO(rtenneti): Enable crashing for Android.
 842 #endif  // OS_ANDROID
 843   }
 844
 845  private:
 846   DISALLOW_COPY_AND_ASSIGN(StartupWatchDogThread);
 847 };
 848
 849 // ShutdownWatchDogThread methods and members.
 850 //
 851 // Class for detecting hangs during shutdown.
 852 class ShutdownWatchDogThread : public base::Watchdog {
 853  public:
 854   // Constructor specifies how long the ShutdownWatchDogThread will wait before
 855   // alarming.
 856   explicit ShutdownWatchDogThread(const base::TimeDelta& duration)
 857       : base::Watchdog(duration, "Shutdown watchdog thread", true) {
 858   }
 859
 860   // Alarm is called if the time expires after an Arm() without someone calling
 861   // Disarm(). We crash the browser if this method is called.
 862   void Alarm() override { metrics::ShutdownHang(); }
 863
 864  private:
 865   DISALLOW_COPY_AND_ASSIGN(ShutdownWatchDogThread);
 866 };
 867 }  // namespace
 868
 869 // StartupTimeBomb methods and members.
 870 //
 871 // static
 872 StartupTimeBomb* StartupTimeBomb::g_startup_timebomb_ = NULL;
 873
 874 StartupTimeBomb::StartupTimeBomb()
 875     : startup_watchdog_(NULL),
 876       thread_id_(base::PlatformThread::CurrentId()) {
 877   CHECK(!g_startup_timebomb_);
 878   g_startup_timebomb_ = this;
 879 }
 880
 881 StartupTimeBomb::~StartupTimeBomb() {
 882   DCHECK(this == g_startup_timebomb_);
 883   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 884   if (startup_watchdog_)
 885     Disarm();
 886   g_startup_timebomb_ = NULL;
 887 }
 888
 889 void StartupTimeBomb::Arm(const base::TimeDelta& duration) {
 890   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 891   DCHECK(!startup_watchdog_);
 892   startup_watchdog_ = new StartupWatchDogThread(duration);
 893   startup_watchdog_->Arm();
 894   return;
 895 }
 896
 897 void StartupTimeBomb::Disarm() {
 898   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 899   if (startup_watchdog_) {
 900     startup_watchdog_->Disarm();
 901     startup_watchdog_->Cleanup();
 902     DeleteStartupWatchdog();
 903   }
 904 }
 905
 906 void StartupTimeBomb::DeleteStartupWatchdog() {
 907   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 908   if (startup_watchdog_->IsJoinable()) {
 909     // Allow the watchdog thread to shutdown on UI. Watchdog thread shutdowns
 910     // very fast.
 911     base::ThreadRestrictions::SetIOAllowed(true);
 912     delete startup_watchdog_;
 913     startup_watchdog_ = NULL;
 914     return;
 915   }
 916   base::MessageLoop::current()->PostDelayedTask(
 917       FROM_HERE,
 918       base::Bind(&StartupTimeBomb::DeleteStartupWatchdog,
 919                  base::Unretained(this)),
 920       base::TimeDelta::FromSeconds(10));
 921 }
 922
 923 // static
 924 void StartupTimeBomb::DisarmStartupTimeBomb() {
 925   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 926   if (g_startup_timebomb_)
 927     g_startup_timebomb_->Disarm();
 928 }
 929
 930 // ShutdownWatcherHelper methods and members.
 931 //
 932 // ShutdownWatcherHelper is a wrapper class for detecting hangs during
 933 // shutdown.
 934 ShutdownWatcherHelper::ShutdownWatcherHelper()
 935     : shutdown_watchdog_(NULL),
 936       thread_id_(base::PlatformThread::CurrentId()) {
 937 }
 938
 939 ShutdownWatcherHelper::~ShutdownWatcherHelper() {
 940   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 941   if (shutdown_watchdog_) {
 942     shutdown_watchdog_->Disarm();
 943     delete shutdown_watchdog_;
 944     shutdown_watchdog_ = NULL;
 945   }
 946 }
 947
 948 void ShutdownWatcherHelper::Arm(const base::TimeDelta& duration) {
 949   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 950   DCHECK(!shutdown_watchdog_);
 951   base::TimeDelta actual_duration = duration;
 952
 953   chrome::VersionInfo::Channel channel = chrome::VersionInfo::GetChannel();
 954   if (channel == chrome::VersionInfo::CHANNEL_STABLE) {
 955     actual_duration *= 20;
 956   } else if (channel == chrome::VersionInfo::CHANNEL_BETA ||
 957              channel == chrome::VersionInfo::CHANNEL_DEV) {
 958     actual_duration *= 10;
 959   }
 960
 961 #if defined(OS_WIN)
 962   // On Windows XP, give twice the time for shutdown.
 963   if (base::win::GetVersion() <= base::win::VERSION_XP)
 964     actual_duration *= 2;
 965 #endif
 966
 967   shutdown_watchdog_ = new ShutdownWatchDogThread(actual_duration);
 968   shutdown_watchdog_->Arm();
 969 }