chrome/browser/metrics/thread_watcher.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/metrics/thread_watcher.h"
   6
   7 #include <math.h>  // ceil
   8
   9 #include "base/bind.h"
  10 #include "base/compiler_specific.h"
  11 #include "base/debug/dump_without_crashing.h"
  12 #include "base/lazy_instance.h"
  13 #include "base/location.h"
  14 #include "base/strings/string_number_conversions.h"
  15 #include "base/strings/string_split.h"
  16 #include "base/strings/string_tokenizer.h"
  17 #include "base/strings/stringprintf.h"
  18 #include "base/thread_task_runner_handle.h"
  19 #include "base/threading/platform_thread.h"
  20 #include "base/threading/thread_restrictions.h"
  21 #include "build/build_config.h"
  22 #include "chrome/browser/chrome_notification_types.h"
  23 #include "chrome/browser/metrics/thread_watcher_report_hang.h"
  24 #include "chrome/common/channel_info.h"
  25 #include "chrome/common/chrome_switches.h"
  26 #include "chrome/common/logging_chrome.h"
  27 #include "components/metrics/call_stack_profile_metrics_provider.h"
  28 #include "components/version_info/version_info.h"
  29 #include "content/public/browser/notification_service.h"
  30
  31 #if defined(OS_WIN)
  32 #include "base/win/windows_version.h"
  33 #endif
  34
  35 using content::BrowserThread;
  36
  37 // ThreadWatcher methods and members.
  38 ThreadWatcher::ThreadWatcher(const WatchingParams& params)
  39     : thread_id_(params.thread_id),
  40       thread_name_(params.thread_name),
  41       watched_runner_(
  42           BrowserThread::GetMessageLoopProxyForThread(params.thread_id)),
  43       sleep_time_(params.sleep_time),
  44       unresponsive_time_(params.unresponsive_time),
  45       ping_time_(base::TimeTicks::Now()),
  46       pong_time_(ping_time_),
  47       ping_sequence_number_(0),
  48       active_(false),
  49       ping_count_(params.unresponsive_threshold),
  50       response_time_histogram_(nullptr),
  51       unresponsive_time_histogram_(nullptr),
  52       unresponsive_count_(0),
  53       hung_processing_complete_(false),
  54       unresponsive_threshold_(params.unresponsive_threshold),
  55       crash_on_hang_(params.crash_on_hang),
  56       live_threads_threshold_(params.live_threads_threshold),
  57       weak_ptr_factory_(this) {
  58   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  59   Initialize();
  60 }
  61
  62 ThreadWatcher::~ThreadWatcher() {}
  63
  64 // static
  65 void ThreadWatcher::StartWatching(const WatchingParams& params) {
  66   DCHECK_GE(params.sleep_time.InMilliseconds(), 0);
  67   DCHECK_GE(params.unresponsive_time.InMilliseconds(),
  68             params.sleep_time.InMilliseconds());
  69
  70   // If we are not on WatchDogThread, then post a task to call StartWatching on
  71   // WatchDogThread.
  72   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
  73     WatchDogThread::PostTask(
  74         FROM_HERE,
  75         base::Bind(&ThreadWatcher::StartWatching, params));
  76     return;
  77   }
  78
  79   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  80
  81   // Create a new thread watcher object for the given thread and activate it.
  82   ThreadWatcher* watcher = new ThreadWatcher(params);
  83
  84   DCHECK(watcher);
  85   // If we couldn't register the thread watcher object, we are shutting down,
  86   // then don't activate thread watching.
  87   if (!ThreadWatcherList::IsRegistered(params.thread_id))
  88     return;
  89   watcher->ActivateThreadWatching();
  90 }
  91
  92 void ThreadWatcher::ActivateThreadWatching() {
  93   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
  94   if (active_) return;
  95   active_ = true;
  96   ping_count_ = unresponsive_threshold_;
  97   ResetHangCounters();
  98   base::ThreadTaskRunnerHandle::Get()->PostTask(
  99       FROM_HERE, base::Bind(&ThreadWatcher::PostPingMessage,
 100                             weak_ptr_factory_.GetWeakPtr()));
 101 }
 102
 103 void ThreadWatcher::DeActivateThreadWatching() {
 104   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 105   active_ = false;
 106   ping_count_ = 0;
 107   weak_ptr_factory_.InvalidateWeakPtrs();
 108 }
 109
 110 void ThreadWatcher::WakeUp() {
 111   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 112   // There is some user activity, PostPingMessage task of thread watcher if
 113   // needed.
 114   if (!active_) return;
 115
 116   // Throw away the previous |unresponsive_count_| and start over again. Just
 117   // before going to sleep, |unresponsive_count_| could be very close to
 118   // |unresponsive_threshold_| and when user becomes active,
 119   // |unresponsive_count_| can go over |unresponsive_threshold_| if there was no
 120   // response for ping messages. Reset |unresponsive_count_| to start measuring
 121   // the unresponsiveness of the threads when system becomes active.
 122   unresponsive_count_ = 0;
 123
 124   if (ping_count_ <= 0) {
 125     ping_count_ = unresponsive_threshold_;
 126     ResetHangCounters();
 127     PostPingMessage();
 128   } else {
 129     ping_count_ = unresponsive_threshold_;
 130   }
 131 }
 132
 133 void ThreadWatcher::PostPingMessage() {
 134   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 135   // If we have stopped watching or if the user is idle, then stop sending
 136   // ping messages.
 137   if (!active_ || ping_count_ <= 0)
 138     return;
 139
 140   // Save the current time when we have sent ping message.
 141   ping_time_ = base::TimeTicks::Now();
 142
 143   // Send a ping message to the watched thread. Callback will be called on
 144   // the WatchDogThread.
 145   base::Closure callback(
 146       base::Bind(&ThreadWatcher::OnPongMessage, weak_ptr_factory_.GetWeakPtr(),
 147                  ping_sequence_number_));
 148   if (watched_runner_->PostTask(
 149           FROM_HERE,
 150           base::Bind(&ThreadWatcher::OnPingMessage, thread_id_,
 151                      callback))) {
 152       // Post a task to check the responsiveness of watched thread.
 153       base::MessageLoop::current()->PostDelayedTask(
 154           FROM_HERE,
 155           base::Bind(&ThreadWatcher::OnCheckResponsiveness,
 156                      weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
 157           unresponsive_time_);
 158   } else {
 159     // Watched thread might have gone away, stop watching it.
 160     DeActivateThreadWatching();
 161   }
 162 }
 163
 164 void ThreadWatcher::OnPongMessage(uint64 ping_sequence_number) {
 165   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 166
 167   // Record watched thread's response time.
 168   base::TimeTicks now = base::TimeTicks::Now();
 169   base::TimeDelta response_time = now - ping_time_;
 170   response_time_histogram_->AddTime(response_time);
 171
 172   // Save the current time when we have got pong message.
 173   pong_time_ = now;
 174
 175   // Check if there are any extra pings in flight.
 176   DCHECK_EQ(ping_sequence_number_, ping_sequence_number);
 177   if (ping_sequence_number_ != ping_sequence_number)
 178     return;
 179
 180   // Increment sequence number for the next ping message to indicate watched
 181   // thread is responsive.
 182   ++ping_sequence_number_;
 183
 184   // If we have stopped watching or if the user is idle, then stop sending
 185   // ping messages.
 186   if (!active_ || --ping_count_ <= 0)
 187     return;
 188
 189   base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 190       FROM_HERE, base::Bind(&ThreadWatcher::PostPingMessage,
 191                             weak_ptr_factory_.GetWeakPtr()),
 192       sleep_time_);
 193 }
 194
 195 void ThreadWatcher::OnCheckResponsiveness(uint64 ping_sequence_number) {
 196   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 197   // If we have stopped watching then consider thread as responding.
 198   if (!active_) {
 199     responsive_ = true;
 200     return;
 201   }
 202   // If the latest ping_sequence_number_ is not same as the ping_sequence_number
 203   // that is passed in, then we can assume OnPongMessage was called.
 204   // OnPongMessage increments ping_sequence_number_.
 205   if (ping_sequence_number_ != ping_sequence_number) {
 206     // Reset unresponsive_count_ to zero because we got a response from the
 207     // watched thread.
 208     ResetHangCounters();
 209
 210     responsive_ = true;
 211     return;
 212   }
 213   // Record that we got no response from watched thread.
 214   GotNoResponse();
 215
 216   // Post a task to check the responsiveness of watched thread.
 217   base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 218       FROM_HERE,
 219       base::Bind(&ThreadWatcher::OnCheckResponsiveness,
 220                  weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
 221       unresponsive_time_);
 222   responsive_ = false;
 223 }
 224
 225 void ThreadWatcher::Initialize() {
 226   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 227   ThreadWatcherList::Register(this);
 228
 229   const std::string response_time_histogram_name =
 230       "ThreadWatcher.ResponseTime." + thread_name_;
 231   response_time_histogram_ = base::Histogram::FactoryTimeGet(
 232       response_time_histogram_name,
 233       base::TimeDelta::FromMilliseconds(1),
 234       base::TimeDelta::FromSeconds(100), 50,
 235       base::Histogram::kUmaTargetedHistogramFlag);
 236
 237   const std::string unresponsive_time_histogram_name =
 238       "ThreadWatcher.Unresponsive." + thread_name_;
 239   unresponsive_time_histogram_ = base::Histogram::FactoryTimeGet(
 240       unresponsive_time_histogram_name,
 241       base::TimeDelta::FromMilliseconds(1),
 242       base::TimeDelta::FromSeconds(100), 50,
 243       base::Histogram::kUmaTargetedHistogramFlag);
 244
 245   const std::string responsive_count_histogram_name =
 246       "ThreadWatcher.ResponsiveThreads." + thread_name_;
 247   responsive_count_histogram_ = base::LinearHistogram::FactoryGet(
 248       responsive_count_histogram_name, 1, 10, 11,
 249       base::Histogram::kUmaTargetedHistogramFlag);
 250
 251   const std::string unresponsive_count_histogram_name =
 252       "ThreadWatcher.UnresponsiveThreads." + thread_name_;
 253   unresponsive_count_histogram_ = base::LinearHistogram::FactoryGet(
 254       unresponsive_count_histogram_name, 1, 10, 11,
 255       base::Histogram::kUmaTargetedHistogramFlag);
 256 }
 257
 258 // static
 259 void ThreadWatcher::OnPingMessage(const BrowserThread::ID& thread_id,
 260                                   const base::Closure& callback_task) {
 261   // This method is called on watched thread.
 262   DCHECK(BrowserThread::CurrentlyOn(thread_id));
 263   WatchDogThread::PostTask(FROM_HERE, callback_task);
 264 }
 265
 266 void ThreadWatcher::ResetHangCounters() {
 267   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 268   unresponsive_count_ = 0;
 269   hung_processing_complete_ = false;
 270 }
 271
 272 void ThreadWatcher::GotNoResponse() {
 273   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 274
 275   ++unresponsive_count_;
 276   if (!IsVeryUnresponsive())
 277     return;
 278
 279   // Record total unresponsive_time since last pong message.
 280   base::TimeDelta unresponse_time = base::TimeTicks::Now() - pong_time_;
 281   unresponsive_time_histogram_->AddTime(unresponse_time);
 282
 283   // We have already collected stats for the non-responding watched thread.
 284   if (hung_processing_complete_)
 285     return;
 286
 287   // Record how other threads are responding.
 288   uint32 responding_thread_count = 0;
 289   uint32 unresponding_thread_count = 0;
 290   ThreadWatcherList::GetStatusOfThreads(&responding_thread_count,
 291                                         &unresponding_thread_count);
 292
 293   // Record how many watched threads are responding.
 294   responsive_count_histogram_->Add(responding_thread_count);
 295
 296   // Record how many watched threads are not responding.
 297   unresponsive_count_histogram_->Add(unresponding_thread_count);
 298
 299   // Crash the browser if the watched thread is to be crashed on hang and if the
 300   // number of other threads responding is less than or equal to
 301   // live_threads_threshold_ and at least one other thread is responding.
 302   if (crash_on_hang_ &&
 303       responding_thread_count > 0 &&
 304       responding_thread_count <= live_threads_threshold_) {
 305     static bool crashed_once = false;
 306     if (!crashed_once) {
 307       crashed_once = true;
 308       metrics::CrashBecauseThreadWasUnresponsive(thread_id_);
 309     }
 310   }
 311
 312   hung_processing_complete_ = true;
 313 }
 314
 315 bool ThreadWatcher::IsVeryUnresponsive() {
 316   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 317   return unresponsive_count_ >= unresponsive_threshold_;
 318 }
 319
 320 // ThreadWatcherList methods and members.
 321 //
 322 // static
 323 ThreadWatcherList* ThreadWatcherList::g_thread_watcher_list_ = nullptr;
 324 // static
 325 bool ThreadWatcherList::g_stopped_ = false;
 326 // static
 327 const int ThreadWatcherList::kSleepSeconds = 1;
 328 // static
 329 const int ThreadWatcherList::kUnresponsiveSeconds = 2;
 330 // static
 331 const int ThreadWatcherList::kUnresponsiveCount = 9;
 332 // static
 333 const int ThreadWatcherList::kLiveThreadsThreshold = 2;
 334 // static, non-const for tests.
 335 int ThreadWatcherList::g_initialize_delay_seconds = 120;
 336
 337 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds(
 338     uint32 live_threads_threshold,
 339     uint32 unresponsive_threshold)
 340     : live_threads_threshold(live_threads_threshold),
 341       unresponsive_threshold(unresponsive_threshold) {
 342 }
 343
 344 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds()
 345     : live_threads_threshold(kLiveThreadsThreshold),
 346       unresponsive_threshold(kUnresponsiveCount) {
 347 }
 348
 349 // static
 350 void ThreadWatcherList::StartWatchingAll(
 351     const base::CommandLine& command_line) {
 352   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 353   uint32 unresponsive_threshold;
 354   CrashOnHangThreadMap crash_on_hang_threads;
 355   ParseCommandLine(command_line,
 356                    &unresponsive_threshold,
 357                    &crash_on_hang_threads);
 358
 359   ThreadWatcherObserver::SetupNotifications(
 360       base::TimeDelta::FromSeconds(kSleepSeconds * unresponsive_threshold));
 361
 362   WatchDogThread::PostTask(
 363       FROM_HERE,
 364       base::Bind(&ThreadWatcherList::SetStopped, false));
 365
 366   if (!WatchDogThread::PostDelayedTask(
 367           FROM_HERE,
 368           base::Bind(&ThreadWatcherList::InitializeAndStartWatching,
 369                      unresponsive_threshold,
 370                      crash_on_hang_threads),
 371           base::TimeDelta::FromSeconds(g_initialize_delay_seconds))) {
 372     // Disarm() the startup timebomb, if we couldn't post the task to start the
 373     // ThreadWatcher (becasue WatchDog thread is not running).
 374     StartupTimeBomb::DisarmStartupTimeBomb();
 375   }
 376 }
 377
 378 // static
 379 void ThreadWatcherList::StopWatchingAll() {
 380   // TODO(rtenneti): Enable ThreadWatcher.
 381   ThreadWatcherObserver::RemoveNotifications();
 382   DeleteAll();
 383 }
 384
 385 // static
 386 void ThreadWatcherList::Register(ThreadWatcher* watcher) {
 387   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 388   if (!g_thread_watcher_list_)
 389     return;
 390   DCHECK(!g_thread_watcher_list_->Find(watcher->thread_id()));
 391   g_thread_watcher_list_->registered_[watcher->thread_id()] = watcher;
 392 }
 393
 394 // static
 395 bool ThreadWatcherList::IsRegistered(const BrowserThread::ID thread_id) {
 396   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 397   return nullptr != ThreadWatcherList::Find(thread_id);
 398 }
 399
 400 // static
 401 void ThreadWatcherList::GetStatusOfThreads(uint32* responding_thread_count,
 402                                            uint32* unresponding_thread_count) {
 403   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 404   *responding_thread_count = 0;
 405   *unresponding_thread_count = 0;
 406   if (!g_thread_watcher_list_)
 407     return;
 408
 409   for (RegistrationList::iterator it =
 410            g_thread_watcher_list_->registered_.begin();
 411        g_thread_watcher_list_->registered_.end() != it;
 412        ++it) {
 413     if (it->second->IsVeryUnresponsive())
 414       ++(*unresponding_thread_count);
 415     else
 416       ++(*responding_thread_count);
 417   }
 418 }
 419
 420 // static
 421 void ThreadWatcherList::WakeUpAll() {
 422   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 423   if (!g_thread_watcher_list_)
 424     return;
 425
 426   for (RegistrationList::iterator it =
 427            g_thread_watcher_list_->registered_.begin();
 428        g_thread_watcher_list_->registered_.end() != it;
 429        ++it)
 430     it->second->WakeUp();
 431 }
 432
 433 ThreadWatcherList::ThreadWatcherList() {
 434   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 435   CHECK(!g_thread_watcher_list_);
 436   g_thread_watcher_list_ = this;
 437 }
 438
 439 ThreadWatcherList::~ThreadWatcherList() {
 440   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 441   DCHECK(this == g_thread_watcher_list_);
 442   g_thread_watcher_list_ = nullptr;
 443 }
 444
 445 // static
 446 void ThreadWatcherList::ParseCommandLine(
 447     const base::CommandLine& command_line,
 448     uint32* unresponsive_threshold,
 449     CrashOnHangThreadMap* crash_on_hang_threads) {
 450   // Initialize |unresponsive_threshold| to a default value.
 451   *unresponsive_threshold = kUnresponsiveCount;
 452
 453   // Increase the unresponsive_threshold on the Stable and Beta channels to
 454   // reduce the number of crashes due to ThreadWatcher.
 455   version_info::Channel channel = chrome::GetChannel();
 456   if (channel == version_info::Channel::STABLE) {
 457     *unresponsive_threshold *= 4;
 458   } else if (channel == version_info::Channel::BETA) {
 459     *unresponsive_threshold *= 2;
 460   }
 461
 462 #if defined(OS_WIN)
 463   // For Windows XP (old systems), double the unresponsive_threshold to give
 464   // the OS a chance to schedule UI/IO threads a time slice to respond with a
 465   // pong message (to get around limitations with the OS).
 466   if (base::win::GetVersion() <= base::win::VERSION_XP)
 467     *unresponsive_threshold *= 2;
 468 #endif
 469
 470   uint32 crash_seconds = *unresponsive_threshold * kUnresponsiveSeconds;
 471   std::string crash_on_hang_thread_names;
 472   if (command_line.HasSwitch(switches::kCrashOnHangThreads)) {
 473     crash_on_hang_thread_names =
 474         command_line.GetSwitchValueASCII(switches::kCrashOnHangThreads);
 475   } else if (channel != version_info::Channel::STABLE) {
 476     // Default to crashing the browser if UI or IO or FILE threads are not
 477     // responsive except in stable channel.
 478     crash_on_hang_thread_names = base::StringPrintf(
 479         "UI:%d:%d,IO:%d:%d,FILE:%d:%d",
 480         kLiveThreadsThreshold, crash_seconds,
 481         kLiveThreadsThreshold, crash_seconds,
 482         kLiveThreadsThreshold, crash_seconds * 5);
 483   }
 484
 485   ParseCommandLineCrashOnHangThreads(crash_on_hang_thread_names,
 486                                      kLiveThreadsThreshold,
 487                                      crash_seconds,
 488                                      crash_on_hang_threads);
 489 }
 490
 491 // static
 492 void ThreadWatcherList::ParseCommandLineCrashOnHangThreads(
 493     const std::string& crash_on_hang_thread_names,
 494     uint32 default_live_threads_threshold,
 495     uint32 default_crash_seconds,
 496     CrashOnHangThreadMap* crash_on_hang_threads) {
 497   base::StringTokenizer tokens(crash_on_hang_thread_names, ",");
 498   while (tokens.GetNext()) {
 499     std::vector<base::StringPiece> values = base::SplitStringPiece(
 500         tokens.token_piece(), ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
 501     std::string thread_name = values[0].as_string();
 502
 503     uint32 live_threads_threshold = default_live_threads_threshold;
 504     uint32 crash_seconds = default_crash_seconds;
 505     if (values.size() >= 2 &&
 506         (!base::StringToUint(values[1], &live_threads_threshold))) {
 507       continue;
 508     }
 509     if (values.size() >= 3 &&
 510         (!base::StringToUint(values[2], &crash_seconds))) {
 511       continue;
 512     }
 513     uint32 unresponsive_threshold = static_cast<uint32>(
 514         ceil(static_cast<float>(crash_seconds) / kUnresponsiveSeconds));
 515
 516     CrashDataThresholds crash_data(live_threads_threshold,
 517                                    unresponsive_threshold);
 518     // Use the last specifier.
 519     (*crash_on_hang_threads)[thread_name] = crash_data;
 520   }
 521 }
 522
 523 // static
 524 void ThreadWatcherList::InitializeAndStartWatching(
 525     uint32 unresponsive_threshold,
 526     const CrashOnHangThreadMap& crash_on_hang_threads) {
 527   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 528
 529   // Disarm the startup timebomb, even if stop has been called.
 530   BrowserThread::PostTask(
 531       BrowserThread::UI,
 532       FROM_HERE,
 533       base::Bind(&StartupTimeBomb::DisarmStartupTimeBomb));
 534
 535   // This method is deferred in relationship to its StopWatchingAll()
 536   // counterpart. If a previous initialization has already happened, or if
 537   // stop has been called, there's nothing left to do here.
 538   if (g_thread_watcher_list_ || g_stopped_)
 539     return;
 540
 541   ThreadWatcherList* thread_watcher_list = new ThreadWatcherList();
 542   CHECK(thread_watcher_list);
 543
 544   // TODO(rtenneti): Because we don't generate crash dumps for ThreadWatcher in
 545   // stable channel, disable ThreadWatcher in stable and unknown channels. We
 546   // will also not collect histogram data in these channels until
 547   // http://crbug.com/426203 is fixed.
 548   version_info::Channel channel = chrome::GetChannel();
 549   if (channel == version_info::Channel::STABLE ||
 550       channel == version_info::Channel::UNKNOWN) {
 551     return;
 552   }
 553
 554   const base::TimeDelta kSleepTime =
 555       base::TimeDelta::FromSeconds(kSleepSeconds);
 556   const base::TimeDelta kUnresponsiveTime =
 557       base::TimeDelta::FromSeconds(kUnresponsiveSeconds);
 558
 559   StartWatching(BrowserThread::UI, "UI", kSleepTime, kUnresponsiveTime,
 560                 unresponsive_threshold, crash_on_hang_threads);
 561   StartWatching(BrowserThread::IO, "IO", kSleepTime, kUnresponsiveTime,
 562                 unresponsive_threshold, crash_on_hang_threads);
 563   StartWatching(BrowserThread::DB, "DB", kSleepTime, kUnresponsiveTime,
 564                 unresponsive_threshold, crash_on_hang_threads);
 565   StartWatching(BrowserThread::FILE, "FILE", kSleepTime, kUnresponsiveTime,
 566                 unresponsive_threshold, crash_on_hang_threads);
 567   StartWatching(BrowserThread::CACHE, "CACHE", kSleepTime, kUnresponsiveTime,
 568                 unresponsive_threshold, crash_on_hang_threads);
 569 }
 570
 571 // static
 572 void ThreadWatcherList::StartWatching(
 573     const BrowserThread::ID& thread_id,
 574     const std::string& thread_name,
 575     const base::TimeDelta& sleep_time,
 576     const base::TimeDelta& unresponsive_time,
 577     uint32 unresponsive_threshold,
 578     const CrashOnHangThreadMap& crash_on_hang_threads) {
 579   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 580
 581   CrashOnHangThreadMap::const_iterator it =
 582       crash_on_hang_threads.find(thread_name);
 583   bool crash_on_hang = false;
 584   uint32 live_threads_threshold = 0;
 585   if (it != crash_on_hang_threads.end()) {
 586     crash_on_hang = true;
 587     live_threads_threshold = it->second.live_threads_threshold;
 588     unresponsive_threshold = it->second.unresponsive_threshold;
 589   }
 590
 591   ThreadWatcher::StartWatching(
 592       ThreadWatcher::WatchingParams(thread_id,
 593                                     thread_name,
 594                                     sleep_time,
 595                                     unresponsive_time,
 596                                     unresponsive_threshold,
 597                                     crash_on_hang,
 598                                     live_threads_threshold));
 599 }
 600
 601 // static
 602 void ThreadWatcherList::DeleteAll() {
 603   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
 604     WatchDogThread::PostTask(
 605         FROM_HERE,
 606         base::Bind(&ThreadWatcherList::DeleteAll));
 607     return;
 608   }
 609
 610   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 611
 612   SetStopped(true);
 613
 614   if (!g_thread_watcher_list_)
 615     return;
 616
 617   // Delete all thread watcher objects.
 618   while (!g_thread_watcher_list_->registered_.empty()) {
 619     RegistrationList::iterator it = g_thread_watcher_list_->registered_.begin();
 620     delete it->second;
 621     g_thread_watcher_list_->registered_.erase(it);
 622   }
 623
 624   delete g_thread_watcher_list_;
 625 }
 626
 627 // static
 628 ThreadWatcher* ThreadWatcherList::Find(const BrowserThread::ID& thread_id) {
 629   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 630   if (!g_thread_watcher_list_)
 631     return nullptr;
 632   RegistrationList::iterator it =
 633       g_thread_watcher_list_->registered_.find(thread_id);
 634   if (g_thread_watcher_list_->registered_.end() == it)
 635     return nullptr;
 636   return it->second;
 637 }
 638
 639 // static
 640 void ThreadWatcherList::SetStopped(bool stopped) {
 641   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 642   g_stopped_ = stopped;
 643 }
 644
 645 // ThreadWatcherObserver methods and members.
 646 //
 647 // static
 648 ThreadWatcherObserver*
 649 ThreadWatcherObserver::g_thread_watcher_observer_ = nullptr;
 650
 651 ThreadWatcherObserver::ThreadWatcherObserver(
 652     const base::TimeDelta& wakeup_interval)
 653     : last_wakeup_time_(base::TimeTicks::Now()),
 654       wakeup_interval_(wakeup_interval) {
 655   CHECK(!g_thread_watcher_observer_);
 656   g_thread_watcher_observer_ = this;
 657 }
 658
 659 ThreadWatcherObserver::~ThreadWatcherObserver() {
 660   DCHECK(this == g_thread_watcher_observer_);
 661   g_thread_watcher_observer_ = nullptr;
 662 }
 663
 664 // static
 665 void ThreadWatcherObserver::SetupNotifications(
 666     const base::TimeDelta& wakeup_interval) {
 667   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 668   ThreadWatcherObserver* observer = new ThreadWatcherObserver(wakeup_interval);
 669   observer->registrar_.Add(
 670       observer,
 671       chrome::NOTIFICATION_BROWSER_OPENED,
 672       content::NotificationService::AllBrowserContextsAndSources());
 673   observer->registrar_.Add(observer,
 674                            chrome::NOTIFICATION_BROWSER_CLOSED,
 675                            content::NotificationService::AllSources());
 676   observer->registrar_.Add(observer,
 677                            chrome::NOTIFICATION_TAB_PARENTED,
 678                            content::NotificationService::AllSources());
 679   observer->registrar_.Add(observer,
 680                            chrome::NOTIFICATION_TAB_CLOSING,
 681                            content::NotificationService::AllSources());
 682   observer->registrar_.Add(observer,
 683                            content::NOTIFICATION_LOAD_START,
 684                            content::NotificationService::AllSources());
 685   observer->registrar_.Add(observer,
 686                            content::NOTIFICATION_LOAD_STOP,
 687                            content::NotificationService::AllSources());
 688   observer->registrar_.Add(observer,
 689                            content::NOTIFICATION_RENDERER_PROCESS_CLOSED,
 690                            content::NotificationService::AllSources());
 691   observer->registrar_.Add(observer,
 692                            content::NOTIFICATION_RENDER_WIDGET_HOST_HANG,
 693                            content::NotificationService::AllSources());
 694   observer->omnibox_url_opened_subscription_ =
 695       OmniboxEventGlobalTracker::GetInstance()->RegisterCallback(
 696           base::Bind(&ThreadWatcherObserver::OnURLOpenedFromOmnibox,
 697                      base::Unretained(observer)));
 698 }
 699
 700 // static
 701 void ThreadWatcherObserver::RemoveNotifications() {
 702   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 703   if (!g_thread_watcher_observer_)
 704     return;
 705   g_thread_watcher_observer_->registrar_.RemoveAll();
 706   delete g_thread_watcher_observer_;
 707 }
 708
 709 void ThreadWatcherObserver::Observe(
 710     int type,
 711     const content::NotificationSource& source,
 712     const content::NotificationDetails& details) {
 713   OnUserActivityDetected();
 714 }
 715
 716 void ThreadWatcherObserver::OnURLOpenedFromOmnibox(OmniboxLog* log) {
 717   OnUserActivityDetected();
 718 }
 719
 720 void ThreadWatcherObserver::OnUserActivityDetected() {
 721   // There is some user activity, see if thread watchers are to be awakened.
 722   base::TimeTicks now = base::TimeTicks::Now();
 723   if ((now - last_wakeup_time_) < wakeup_interval_)
 724     return;
 725   last_wakeup_time_ = now;
 726   WatchDogThread::PostTask(
 727       FROM_HERE,
 728       base::Bind(&ThreadWatcherList::WakeUpAll));
 729 }
 730
 731 // WatchDogThread methods and members.
 732
 733 // This lock protects g_watchdog_thread.
 734 static base::LazyInstance<base::Lock>::Leaky
 735     g_watchdog_lock = LAZY_INSTANCE_INITIALIZER;
 736
 737 // The singleton of this class.
 738 static WatchDogThread* g_watchdog_thread = nullptr;
 739
 740 WatchDogThread::WatchDogThread() : Thread("BrowserWatchdog") {
 741 }
 742
 743 WatchDogThread::~WatchDogThread() {
 744   Stop();
 745 }
 746
 747 // static
 748 bool WatchDogThread::CurrentlyOnWatchDogThread() {
 749   base::AutoLock lock(g_watchdog_lock.Get());
 750   return g_watchdog_thread &&
 751       g_watchdog_thread->message_loop() == base::MessageLoop::current();
 752 }
 753
 754 // static
 755 bool WatchDogThread::PostTask(const tracked_objects::Location& from_here,
 756                               const base::Closure& task) {
 757   return PostTaskHelper(from_here, task, base::TimeDelta());
 758 }
 759
 760 // static
 761 bool WatchDogThread::PostDelayedTask(const tracked_objects::Location& from_here,
 762                                      const base::Closure& task,
 763                                      base::TimeDelta delay) {
 764   return PostTaskHelper(from_here, task, delay);
 765 }
 766
 767 // static
 768 bool WatchDogThread::PostTaskHelper(
 769     const tracked_objects::Location& from_here,
 770     const base::Closure& task,
 771     base::TimeDelta delay) {
 772   {
 773     base::AutoLock lock(g_watchdog_lock.Get());
 774
 775     base::MessageLoop* message_loop = g_watchdog_thread ?
 776         g_watchdog_thread->message_loop() : nullptr;
 777     if (message_loop) {
 778       message_loop->task_runner()->PostDelayedTask(from_here, task, delay);
 779       return true;
 780     }
 781   }
 782
 783   return false;
 784 }
 785
 786 bool WatchDogThread::Started() const {
 787   base::AutoLock lock(g_watchdog_lock.Get());
 788   return g_watchdog_thread != nullptr;
 789 }
 790
 791 void WatchDogThread::Init() {
 792   // This thread shouldn't be allowed to perform any blocking disk I/O.
 793   base::ThreadRestrictions::SetIOAllowed(false);
 794
 795   base::AutoLock lock(g_watchdog_lock.Get());
 796   CHECK(!g_watchdog_thread);
 797   g_watchdog_thread = this;
 798 }
 799
 800 void WatchDogThread::CleanUp() {
 801   base::AutoLock lock(g_watchdog_lock.Get());
 802   g_watchdog_thread = nullptr;
 803 }
 804
 805 namespace {
 806
 807 // StartupWatchDogThread methods and members.
 808 //
 809 // Class for detecting hangs during startup.
 810 class StartupWatchDogThread : public base::Watchdog {
 811  public:
 812   // Constructor specifies how long the StartupWatchDogThread will wait before
 813   // alarming.
 814   explicit StartupWatchDogThread(const base::TimeDelta& duration)
 815       : base::Watchdog(duration, "Startup watchdog thread", true) {
 816   }
 817
 818   // Alarm is called if the time expires after an Arm() without someone calling
 819   // Disarm(). When Alarm goes off, in release mode we get the crash dump
 820   // without crashing and in debug mode we break into the debugger.
 821   void Alarm() override {
 822 #if !defined(NDEBUG)
 823     metrics::StartupHang();
 824     return;
 825 #elif !defined(OS_ANDROID)
 826     WatchDogThread::PostTask(FROM_HERE, base::Bind(&metrics::StartupHang));
 827     return;
 828 #else
 829     // TODO(rtenneti): Enable crashing for Android.
 830 #endif  // OS_ANDROID
 831   }
 832
 833  private:
 834   DISALLOW_COPY_AND_ASSIGN(StartupWatchDogThread);
 835 };
 836
 837 // ShutdownWatchDogThread methods and members.
 838 //
 839 // Class for detecting hangs during shutdown.
 840 class ShutdownWatchDogThread : public base::Watchdog {
 841  public:
 842   // Constructor specifies how long the ShutdownWatchDogThread will wait before
 843   // alarming.
 844   explicit ShutdownWatchDogThread(const base::TimeDelta& duration)
 845       : base::Watchdog(duration, "Shutdown watchdog thread", true) {
 846   }
 847
 848   // Alarm is called if the time expires after an Arm() without someone calling
 849   // Disarm(). We crash the browser if this method is called.
 850   void Alarm() override { metrics::ShutdownHang(); }
 851
 852  private:
 853   DISALLOW_COPY_AND_ASSIGN(ShutdownWatchDogThread);
 854 };
 855
 856 }  // namespace
 857
 858 // StartupTimeBomb methods and members.
 859 //
 860 // static
 861 StartupTimeBomb* StartupTimeBomb::g_startup_timebomb_ = nullptr;
 862
 863 StartupTimeBomb::StartupTimeBomb()
 864     : startup_watchdog_(nullptr),
 865       thread_id_(base::PlatformThread::CurrentId()) {
 866   CHECK(!g_startup_timebomb_);
 867   g_startup_timebomb_ = this;
 868 }
 869
 870 StartupTimeBomb::~StartupTimeBomb() {
 871   DCHECK(this == g_startup_timebomb_);
 872   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 873   if (startup_watchdog_)
 874     Disarm();
 875   g_startup_timebomb_ = nullptr;
 876 }
 877
 878 void StartupTimeBomb::Arm(const base::TimeDelta& duration) {
 879   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 880   DCHECK(!startup_watchdog_);
 881   startup_watchdog_ = new StartupWatchDogThread(duration);
 882   startup_watchdog_->Arm();
 883   return;
 884 }
 885
 886 void StartupTimeBomb::Disarm() {
 887   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 888   if (startup_watchdog_) {
 889     startup_watchdog_->Disarm();
 890     startup_watchdog_->Cleanup();
 891     DeleteStartupWatchdog();
 892   }
 893 }
 894
 895 void StartupTimeBomb::DeleteStartupWatchdog() {
 896   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 897   if (startup_watchdog_->IsJoinable()) {
 898     // Allow the watchdog thread to shutdown on UI. Watchdog thread shutdowns
 899     // very fast.
 900     base::ThreadRestrictions::SetIOAllowed(true);
 901     delete startup_watchdog_;
 902     startup_watchdog_ = nullptr;
 903     return;
 904   }
 905   base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 906       FROM_HERE, base::Bind(&StartupTimeBomb::DeleteStartupWatchdog,
 907                             base::Unretained(this)),
 908       base::TimeDelta::FromSeconds(10));
 909 }
 910
 911 // static
 912 void StartupTimeBomb::DisarmStartupTimeBomb() {
 913   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 914   if (g_startup_timebomb_)
 915     g_startup_timebomb_->Disarm();
 916 }
 917
 918 base::StackSamplingProfiler::SamplingParams GetJankTimeBombSamplingParams() {
 919   base::StackSamplingProfiler::SamplingParams params;
 920   params.initial_delay = base::TimeDelta::FromMilliseconds(0);
 921   params.bursts = 1;
 922   // 5 seconds at 10Hz.
 923   params.samples_per_burst = 50;
 924   params.sampling_interval = base::TimeDelta::FromMilliseconds(100);
 925   return params;
 926 }
 927
 928 // JankTimeBomb methods and members.
 929 //
 930 JankTimeBomb::JankTimeBomb(base::TimeDelta duration)
 931     : weak_ptr_factory_(this) {
 932   if (IsEnabled()) {
 933     WatchDogThread::PostDelayedTask(
 934         FROM_HERE,
 935         base::Bind(&JankTimeBomb::Alarm,
 936                    weak_ptr_factory_.GetWeakPtr(),
 937                    base::PlatformThread::CurrentId()),
 938         duration);
 939   }
 940 }
 941
 942 JankTimeBomb::~JankTimeBomb() {
 943 }
 944
 945 bool JankTimeBomb::IsEnabled() const {
 946   version_info::Channel channel = chrome::GetChannel();
 947   return channel == version_info::Channel::UNKNOWN ||
 948       channel == version_info::Channel::CANARY ||
 949       channel == version_info::Channel::DEV;
 950 }
 951
 952 void JankTimeBomb::Alarm(base::PlatformThreadId thread_id) {
 953   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 954   sampling_profiler_.reset(new base::StackSamplingProfiler(
 955       thread_id,
 956       GetJankTimeBombSamplingParams(),
 957       metrics::CallStackProfileMetricsProvider::GetProfilerCallback(
 958           metrics::CallStackProfileMetricsProvider::Params(
 959               metrics::CallStackProfileMetricsProvider::JANKY_TASK,
 960               true))));
 961   // Use synchronous profiler. It will automatically stop collection when
 962   // destroyed.
 963   sampling_profiler_->Start();
 964 }
 965
 966 // ShutdownWatcherHelper methods and members.
 967 //
 968 // ShutdownWatcherHelper is a wrapper class for detecting hangs during
 969 // shutdown.
 970 ShutdownWatcherHelper::ShutdownWatcherHelper()
 971     : shutdown_watchdog_(nullptr),
 972       thread_id_(base::PlatformThread::CurrentId()) {
 973 }
 974
 975 ShutdownWatcherHelper::~ShutdownWatcherHelper() {
 976   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 977   if (shutdown_watchdog_) {
 978     shutdown_watchdog_->Disarm();
 979     delete shutdown_watchdog_;
 980     shutdown_watchdog_ = nullptr;
 981   }
 982 }
 983
 984 void ShutdownWatcherHelper::Arm(const base::TimeDelta& duration) {
 985   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
 986   DCHECK(!shutdown_watchdog_);
 987   base::TimeDelta actual_duration = duration;
 988
 989   version_info::Channel channel = chrome::GetChannel();
 990   if (channel == version_info::Channel::STABLE) {
 991     actual_duration *= 20;
 992   } else if (channel == version_info::Channel::BETA ||
 993              channel == version_info::Channel::DEV) {
 994     actual_duration *= 10;
 995   }
 996
 997 #if defined(OS_WIN)
 998   // On Windows XP, give twice the time for shutdown.
 999   if (base::win::GetVersion() <= base::win::VERSION_XP)
1000     actual_duration *= 2;
1001 #endif
1002
1003   shutdown_watchdog_ = new ShutdownWatchDogThread(actual_duration);
1004   shutdown_watchdog_->Arm();
1005 }