net/socket/client_socket_pool_base.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/socket/client_socket_pool_base.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/compiler_specific.h"
  10 #include "base/format_macros.h"
  11 #include "base/logging.h"
  12 #include "base/message_loop/message_loop.h"
  13 #include "base/stl_util.h"
  14 #include "base/strings/string_util.h"
  15 #include "base/time/time.h"
  16 #include "base/values.h"
  17 #include "net/base/net_errors.h"
  18 #include "net/log/net_log.h"
  19
  20 using base::TimeDelta;
  21
  22 namespace net {
  23
  24 namespace {
  25
  26 // Indicate whether we should enable idle socket cleanup timer. When timer is
  27 // disabled, sockets are closed next time a socket request is made.
  28 bool g_cleanup_timer_enabled = true;
  29
  30 // The timeout value, in seconds, used to clean up idle sockets that can't be
  31 // reused.
  32 //
  33 // Note: It's important to close idle sockets that have received data as soon
  34 // as possible because the received data may cause BSOD on Windows XP under
  35 // some conditions.  See http://crbug.com/4606.
  36 const int kCleanupInterval = 10;  // DO NOT INCREASE THIS TIMEOUT.
  37
  38 // Indicate whether or not we should establish a new transport layer connection
  39 // after a certain timeout has passed without receiving an ACK.
  40 bool g_connect_backup_jobs_enabled = true;
  41
  42 }  // namespace
  43
  44 ConnectJob::ConnectJob(const std::string& group_name,
  45                        base::TimeDelta timeout_duration,
  46                        RequestPriority priority,
  47                        Delegate* delegate,
  48                        const BoundNetLog& net_log)
  49     : group_name_(group_name),
  50       timeout_duration_(timeout_duration),
  51       priority_(priority),
  52       delegate_(delegate),
  53       net_log_(net_log),
  54       idle_(true) {
  55   DCHECK(!group_name.empty());
  56   DCHECK(delegate);
  57   net_log.BeginEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB,
  58                      NetLog::StringCallback("group_name", &group_name_));
  59 }
  60
  61 ConnectJob::~ConnectJob() {
  62   net_log().EndEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB);
  63 }
  64
  65 scoped_ptr<StreamSocket> ConnectJob::PassSocket() {
  66   return socket_.Pass();
  67 }
  68
  69 int ConnectJob::Connect() {
  70   if (timeout_duration_ != base::TimeDelta())
  71     timer_.Start(FROM_HERE, timeout_duration_, this, &ConnectJob::OnTimeout);
  72
  73   idle_ = false;
  74
  75   LogConnectStart();
  76
  77   int rv = ConnectInternal();
  78
  79   if (rv != ERR_IO_PENDING) {
  80     LogConnectCompletion(rv);
  81     delegate_ = NULL;
  82   }
  83
  84   return rv;
  85 }
  86
  87 void ConnectJob::SetSocket(scoped_ptr<StreamSocket> socket) {
  88   if (socket) {
  89     net_log().AddEvent(NetLog::TYPE_CONNECT_JOB_SET_SOCKET,
  90                        socket->NetLog().source().ToEventParametersCallback());
  91   }
  92   socket_ = socket.Pass();
  93 }
  94
  95 void ConnectJob::NotifyDelegateOfCompletion(int rv) {
  96   // The delegate will own |this|.
  97   Delegate* delegate = delegate_;
  98   delegate_ = NULL;
  99
 100   LogConnectCompletion(rv);
 101   delegate->OnConnectJobComplete(rv, this);
 102 }
 103
 104 void ConnectJob::ResetTimer(base::TimeDelta remaining_time) {
 105   timer_.Stop();
 106   timer_.Start(FROM_HERE, remaining_time, this, &ConnectJob::OnTimeout);
 107 }
 108
 109 void ConnectJob::LogConnectStart() {
 110   connect_timing_.connect_start = base::TimeTicks::Now();
 111   net_log().BeginEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_CONNECT);
 112 }
 113
 114 void ConnectJob::LogConnectCompletion(int net_error) {
 115   connect_timing_.connect_end = base::TimeTicks::Now();
 116   net_log().EndEventWithNetErrorCode(
 117       NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_CONNECT, net_error);
 118 }
 119
 120 void ConnectJob::OnTimeout() {
 121   // Make sure the socket is NULL before calling into |delegate|.
 122   SetSocket(scoped_ptr<StreamSocket>());
 123
 124   net_log_.AddEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_TIMED_OUT);
 125
 126   NotifyDelegateOfCompletion(ERR_TIMED_OUT);
 127 }
 128
 129 namespace internal {
 130
 131 ClientSocketPoolBaseHelper::Request::Request(
 132     ClientSocketHandle* handle,
 133     const CompletionCallback& callback,
 134     RequestPriority priority,
 135     bool ignore_limits,
 136     Flags flags,
 137     const BoundNetLog& net_log)
 138     : handle_(handle),
 139       callback_(callback),
 140       priority_(priority),
 141       ignore_limits_(ignore_limits),
 142       flags_(flags),
 143       net_log_(net_log) {
 144   if (ignore_limits_)
 145     DCHECK_EQ(priority_, MAXIMUM_PRIORITY);
 146 }
 147
 148 ClientSocketPoolBaseHelper::Request::~Request() {
 149   liveness_ = DEAD;
 150 }
 151
 152 void ClientSocketPoolBaseHelper::Request::CrashIfInvalid() const {
 153   CHECK_EQ(liveness_, ALIVE);
 154 }
 155
 156 ClientSocketPoolBaseHelper::ClientSocketPoolBaseHelper(
 157     HigherLayeredPool* pool,
 158     int max_sockets,
 159     int max_sockets_per_group,
 160     base::TimeDelta unused_idle_socket_timeout,
 161     base::TimeDelta used_idle_socket_timeout,
 162     ConnectJobFactory* connect_job_factory)
 163     : idle_socket_count_(0),
 164       connecting_socket_count_(0),
 165       handed_out_socket_count_(0),
 166       max_sockets_(max_sockets),
 167       max_sockets_per_group_(max_sockets_per_group),
 168       use_cleanup_timer_(g_cleanup_timer_enabled),
 169       unused_idle_socket_timeout_(unused_idle_socket_timeout),
 170       used_idle_socket_timeout_(used_idle_socket_timeout),
 171       connect_job_factory_(connect_job_factory),
 172       connect_backup_jobs_enabled_(false),
 173       pool_generation_number_(0),
 174       pool_(pool),
 175       weak_factory_(this) {
 176   DCHECK_LE(0, max_sockets_per_group);
 177   DCHECK_LE(max_sockets_per_group, max_sockets);
 178
 179   NetworkChangeNotifier::AddIPAddressObserver(this);
 180 }
 181
 182 ClientSocketPoolBaseHelper::~ClientSocketPoolBaseHelper() {
 183   // Clean up any idle sockets and pending connect jobs.  Assert that we have no
 184   // remaining active sockets or pending requests.  They should have all been
 185   // cleaned up prior to |this| being destroyed.
 186   FlushWithError(ERR_ABORTED);
 187   DCHECK(group_map_.empty());
 188   DCHECK(pending_callback_map_.empty());
 189   DCHECK_EQ(0, connecting_socket_count_);
 190   CHECK(higher_pools_.empty());
 191
 192   NetworkChangeNotifier::RemoveIPAddressObserver(this);
 193
 194   // Remove from lower layer pools.
 195   for (std::set<LowerLayeredPool*>::iterator it = lower_pools_.begin();
 196        it != lower_pools_.end();
 197        ++it) {
 198     (*it)->RemoveHigherLayeredPool(pool_);
 199   }
 200 }
 201
 202 ClientSocketPoolBaseHelper::CallbackResultPair::CallbackResultPair()
 203     : result(OK) {
 204 }
 205
 206 ClientSocketPoolBaseHelper::CallbackResultPair::CallbackResultPair(
 207     const CompletionCallback& callback_in, int result_in)
 208     : callback(callback_in),
 209       result(result_in) {
 210 }
 211
 212 ClientSocketPoolBaseHelper::CallbackResultPair::~CallbackResultPair() {}
 213
 214 bool ClientSocketPoolBaseHelper::IsStalled() const {
 215   // If a lower layer pool is stalled, consider |this| stalled as well.
 216   for (std::set<LowerLayeredPool*>::const_iterator it = lower_pools_.begin();
 217        it != lower_pools_.end();
 218        ++it) {
 219     if ((*it)->IsStalled())
 220       return true;
 221   }
 222
 223   // If fewer than |max_sockets_| are in use, then clearly |this| is not
 224   // stalled.
 225   if ((handed_out_socket_count_ + connecting_socket_count_) < max_sockets_)
 226     return false;
 227   // So in order to be stalled, |this| must be using at least |max_sockets_| AND
 228   // |this| must have a request that is actually stalled on the global socket
 229   // limit.  To find such a request, look for a group that has more requests
 230   // than jobs AND where the number of sockets is less than
 231   // |max_sockets_per_group_|.  (If the number of sockets is equal to
 232   // |max_sockets_per_group_|, then the request is stalled on the group limit,
 233   // which does not count.)
 234   for (GroupMap::const_iterator it = group_map_.begin();
 235        it != group_map_.end(); ++it) {
 236     if (it->second->CanUseAdditionalSocketSlot(max_sockets_per_group_))
 237       return true;
 238   }
 239   return false;
 240 }
 241
 242 void ClientSocketPoolBaseHelper::AddLowerLayeredPool(
 243     LowerLayeredPool* lower_pool) {
 244   DCHECK(pool_);
 245   CHECK(!ContainsKey(lower_pools_, lower_pool));
 246   lower_pools_.insert(lower_pool);
 247   lower_pool->AddHigherLayeredPool(pool_);
 248 }
 249
 250 void ClientSocketPoolBaseHelper::AddHigherLayeredPool(
 251     HigherLayeredPool* higher_pool) {
 252   CHECK(higher_pool);
 253   CHECK(!ContainsKey(higher_pools_, higher_pool));
 254   higher_pools_.insert(higher_pool);
 255 }
 256
 257 void ClientSocketPoolBaseHelper::RemoveHigherLayeredPool(
 258     HigherLayeredPool* higher_pool) {
 259   CHECK(higher_pool);
 260   CHECK(ContainsKey(higher_pools_, higher_pool));
 261   higher_pools_.erase(higher_pool);
 262 }
 263
 264 int ClientSocketPoolBaseHelper::RequestSocket(
 265     const std::string& group_name,
 266     scoped_ptr<const Request> request) {
 267   CHECK(!request->callback().is_null());
 268   CHECK(request->handle());
 269
 270   // Cleanup any timed-out idle sockets if no timer is used.
 271   if (!use_cleanup_timer_)
 272     CleanupIdleSockets(false);
 273
 274   request->net_log().BeginEvent(NetLog::TYPE_SOCKET_POOL);
 275   Group* group = GetOrCreateGroup(group_name);
 276
 277   int rv = RequestSocketInternal(group_name, *request);
 278   if (rv != ERR_IO_PENDING) {
 279     request->net_log().EndEventWithNetErrorCode(NetLog::TYPE_SOCKET_POOL, rv);
 280     CHECK(!request->handle()->is_initialized());
 281     request.reset();
 282   } else {
 283     group->InsertPendingRequest(request.Pass());
 284     // Have to do this asynchronously, as closing sockets in higher level pools
 285     // call back in to |this|, which will cause all sorts of fun and exciting
 286     // re-entrancy issues if the socket pool is doing something else at the
 287     // time.
 288     if (group->CanUseAdditionalSocketSlot(max_sockets_per_group_)) {
 289       base::MessageLoop::current()->PostTask(
 290           FROM_HERE,
 291           base::Bind(
 292               &ClientSocketPoolBaseHelper::TryToCloseSocketsInLayeredPools,
 293               weak_factory_.GetWeakPtr()));
 294     }
 295   }
 296   return rv;
 297 }
 298
 299 void ClientSocketPoolBaseHelper::RequestSockets(
 300     const std::string& group_name,
 301     const Request& request,
 302     int num_sockets) {
 303   DCHECK(request.callback().is_null());
 304   DCHECK(!request.handle());
 305
 306   // Cleanup any timed out idle sockets if no timer is used.
 307   if (!use_cleanup_timer_)
 308     CleanupIdleSockets(false);
 309
 310   if (num_sockets > max_sockets_per_group_) {
 311     num_sockets = max_sockets_per_group_;
 312   }
 313
 314   request.net_log().BeginEvent(
 315       NetLog::TYPE_SOCKET_POOL_CONNECTING_N_SOCKETS,
 316       NetLog::IntegerCallback("num_sockets", num_sockets));
 317
 318   Group* group = GetOrCreateGroup(group_name);
 319
 320   // RequestSocketsInternal() may delete the group.
 321   bool deleted_group = false;
 322
 323   int rv = OK;
 324   for (int num_iterations_left = num_sockets;
 325        group->NumActiveSocketSlots() < num_sockets &&
 326        num_iterations_left > 0 ; num_iterations_left--) {
 327     rv = RequestSocketInternal(group_name, request);
 328     if (rv < 0 && rv != ERR_IO_PENDING) {
 329       // We're encountering a synchronous error.  Give up.
 330       if (!ContainsKey(group_map_, group_name))
 331         deleted_group = true;
 332       break;
 333     }
 334     if (!ContainsKey(group_map_, group_name)) {
 335       // Unexpected.  The group should only be getting deleted on synchronous
 336       // error.
 337       NOTREACHED();
 338       deleted_group = true;
 339       break;
 340     }
 341   }
 342
 343   if (!deleted_group && group->IsEmpty())
 344     RemoveGroup(group_name);
 345
 346   if (rv == ERR_IO_PENDING)
 347     rv = OK;
 348   request.net_log().EndEventWithNetErrorCode(
 349       NetLog::TYPE_SOCKET_POOL_CONNECTING_N_SOCKETS, rv);
 350 }
 351
 352 int ClientSocketPoolBaseHelper::RequestSocketInternal(
 353     const std::string& group_name,
 354     const Request& request) {
 355   ClientSocketHandle* const handle = request.handle();
 356   const bool preconnecting = !handle;
 357   Group* group = GetOrCreateGroup(group_name);
 358
 359   if (!(request.flags() & NO_IDLE_SOCKETS)) {
 360     // Try to reuse a socket.
 361     if (AssignIdleSocketToRequest(request, group))
 362       return OK;
 363   }
 364
 365   // If there are more ConnectJobs than pending requests, don't need to do
 366   // anything.  Can just wait for the extra job to connect, and then assign it
 367   // to the request.
 368   if (!preconnecting && group->TryToUseUnassignedConnectJob())
 369     return ERR_IO_PENDING;
 370
 371   // Can we make another active socket now?
 372   if (!group->HasAvailableSocketSlot(max_sockets_per_group_) &&
 373       !request.ignore_limits()) {
 374     // TODO(willchan): Consider whether or not we need to close a socket in a
 375     // higher layered group. I don't think this makes sense since we would just
 376     // reuse that socket then if we needed one and wouldn't make it down to this
 377     // layer.
 378     request.net_log().AddEvent(
 379         NetLog::TYPE_SOCKET_POOL_STALLED_MAX_SOCKETS_PER_GROUP);
 380     return ERR_IO_PENDING;
 381   }
 382
 383   if (ReachedMaxSocketsLimit() && !request.ignore_limits()) {
 384     // NOTE(mmenke):  Wonder if we really need different code for each case
 385     // here.  Only reason for them now seems to be preconnects.
 386     if (idle_socket_count() > 0) {
 387       // There's an idle socket in this pool. Either that's because there's
 388       // still one in this group, but we got here due to preconnecting bypassing
 389       // idle sockets, or because there's an idle socket in another group.
 390       bool closed = CloseOneIdleSocketExceptInGroup(group);
 391       if (preconnecting && !closed)
 392         return ERR_PRECONNECT_MAX_SOCKET_LIMIT;
 393     } else {
 394       // We could check if we really have a stalled group here, but it requires
 395       // a scan of all groups, so just flip a flag here, and do the check later.
 396       request.net_log().AddEvent(NetLog::TYPE_SOCKET_POOL_STALLED_MAX_SOCKETS);
 397       return ERR_IO_PENDING;
 398     }
 399   }
 400
 401   // We couldn't find a socket to reuse, and there's space to allocate one,
 402   // so allocate and connect a new one.
 403   scoped_ptr<ConnectJob> connect_job(
 404       connect_job_factory_->NewConnectJob(group_name, request, this));
 405
 406   int rv = connect_job->Connect();
 407   if (rv == OK) {
 408     LogBoundConnectJobToRequest(connect_job->net_log().source(), request);
 409     if (!preconnecting) {
 410       HandOutSocket(connect_job->PassSocket(), ClientSocketHandle::UNUSED,
 411                     connect_job->connect_timing(), handle, base::TimeDelta(),
 412                     group, request.net_log());
 413     } else {
 414       AddIdleSocket(connect_job->PassSocket(), group);
 415     }
 416   } else if (rv == ERR_IO_PENDING) {
 417     // If we don't have any sockets in this group, set a timer for potentially
 418     // creating a new one.  If the SYN is lost, this backup socket may complete
 419     // before the slow socket, improving end user latency.
 420     if (connect_backup_jobs_enabled_ && group->IsEmpty()) {
 421       group->StartBackupJobTimer(group_name, this);
 422     }
 423
 424     connecting_socket_count_++;
 425
 426     group->AddJob(connect_job.Pass(), preconnecting);
 427   } else {
 428     LogBoundConnectJobToRequest(connect_job->net_log().source(), request);
 429     scoped_ptr<StreamSocket> error_socket;
 430     if (!preconnecting) {
 431       DCHECK(handle);
 432       connect_job->GetAdditionalErrorState(handle);
 433       error_socket = connect_job->PassSocket();
 434     }
 435     if (error_socket) {
 436       HandOutSocket(error_socket.Pass(), ClientSocketHandle::UNUSED,
 437                     connect_job->connect_timing(), handle, base::TimeDelta(),
 438                     group, request.net_log());
 439     } else if (group->IsEmpty()) {
 440       RemoveGroup(group_name);
 441     }
 442   }
 443
 444   return rv;
 445 }
 446
 447 bool ClientSocketPoolBaseHelper::AssignIdleSocketToRequest(
 448     const Request& request, Group* group) {
 449   std::list<IdleSocket>* idle_sockets = group->mutable_idle_sockets();
 450   std::list<IdleSocket>::iterator idle_socket_it = idle_sockets->end();
 451
 452   // Iterate through the idle sockets forwards (oldest to newest)
 453   //   * Delete any disconnected ones.
 454   //   * If we find a used idle socket, assign to |idle_socket|.  At the end,
 455   //   the |idle_socket_it| will be set to the newest used idle socket.
 456   for (std::list<IdleSocket>::iterator it = idle_sockets->begin();
 457        it != idle_sockets->end();) {
 458     if (!it->IsUsable()) {
 459       DecrementIdleCount();
 460       delete it->socket;
 461       it = idle_sockets->erase(it);
 462       continue;
 463     }
 464
 465     if (it->socket->WasEverUsed()) {
 466       // We found one we can reuse!
 467       idle_socket_it = it;
 468     }
 469
 470     ++it;
 471   }
 472
 473   // If we haven't found an idle socket, that means there are no used idle
 474   // sockets.  Pick the oldest (first) idle socket (FIFO).
 475
 476   if (idle_socket_it == idle_sockets->end() && !idle_sockets->empty())
 477     idle_socket_it = idle_sockets->begin();
 478
 479   if (idle_socket_it != idle_sockets->end()) {
 480     DecrementIdleCount();
 481     base::TimeDelta idle_time =
 482         base::TimeTicks::Now() - idle_socket_it->start_time;
 483     IdleSocket idle_socket = *idle_socket_it;
 484     idle_sockets->erase(idle_socket_it);
 485     // TODO(davidben): If |idle_time| is under some low watermark, consider
 486     // treating as UNUSED rather than UNUSED_IDLE. This will avoid
 487     // HttpNetworkTransaction retrying on some errors.
 488     ClientSocketHandle::SocketReuseType reuse_type =
 489         idle_socket.socket->WasEverUsed() ?
 490             ClientSocketHandle::REUSED_IDLE :
 491             ClientSocketHandle::UNUSED_IDLE;
 492
 493     // If this socket took multiple attempts to obtain, don't report those
 494     // every time it's reused, just to the first user.
 495     if (idle_socket.socket->WasEverUsed())
 496       idle_socket.socket->ClearConnectionAttempts();
 497
 498     HandOutSocket(
 499         scoped_ptr<StreamSocket>(idle_socket.socket),
 500         reuse_type,
 501         LoadTimingInfo::ConnectTiming(),
 502         request.handle(),
 503         idle_time,
 504         group,
 505         request.net_log());
 506     return true;
 507   }
 508
 509   return false;
 510 }
 511
 512 // static
 513 void ClientSocketPoolBaseHelper::LogBoundConnectJobToRequest(
 514     const NetLog::Source& connect_job_source, const Request& request) {
 515   request.net_log().AddEvent(NetLog::TYPE_SOCKET_POOL_BOUND_TO_CONNECT_JOB,
 516                              connect_job_source.ToEventParametersCallback());
 517 }
 518
 519 void ClientSocketPoolBaseHelper::CancelRequest(
 520     const std::string& group_name, ClientSocketHandle* handle) {
 521   PendingCallbackMap::iterator callback_it = pending_callback_map_.find(handle);
 522   if (callback_it != pending_callback_map_.end()) {
 523     int result = callback_it->second.result;
 524     pending_callback_map_.erase(callback_it);
 525     scoped_ptr<StreamSocket> socket = handle->PassSocket();
 526     if (socket) {
 527       if (result != OK)
 528         socket->Disconnect();
 529       ReleaseSocket(handle->group_name(), socket.Pass(), handle->id());
 530     }
 531     return;
 532   }
 533
 534   CHECK(ContainsKey(group_map_, group_name));
 535
 536   Group* group = GetOrCreateGroup(group_name);
 537
 538   // Search pending_requests for matching handle.
 539   scoped_ptr<const Request> request =
 540       group->FindAndRemovePendingRequest(handle);
 541   if (request) {
 542     request->net_log().AddEvent(NetLog::TYPE_CANCELLED);
 543     request->net_log().EndEvent(NetLog::TYPE_SOCKET_POOL);
 544
 545     // We let the job run, unless we're at the socket limit and there is
 546     // not another request waiting on the job.
 547     if (group->jobs().size() > group->pending_request_count() &&
 548         ReachedMaxSocketsLimit()) {
 549       RemoveConnectJob(*group->jobs().begin(), group);
 550       CheckForStalledSocketGroups();
 551     }
 552   }
 553 }
 554
 555 bool ClientSocketPoolBaseHelper::HasGroup(const std::string& group_name) const {
 556   return ContainsKey(group_map_, group_name);
 557 }
 558
 559 void ClientSocketPoolBaseHelper::CloseIdleSockets() {
 560   CleanupIdleSockets(true);
 561   DCHECK_EQ(0, idle_socket_count_);
 562 }
 563
 564 int ClientSocketPoolBaseHelper::IdleSocketCountInGroup(
 565     const std::string& group_name) const {
 566   GroupMap::const_iterator i = group_map_.find(group_name);
 567   CHECK(i != group_map_.end());
 568
 569   return i->second->idle_sockets().size();
 570 }
 571
 572 LoadState ClientSocketPoolBaseHelper::GetLoadState(
 573     const std::string& group_name,
 574     const ClientSocketHandle* handle) const {
 575   if (ContainsKey(pending_callback_map_, handle))
 576     return LOAD_STATE_CONNECTING;
 577
 578   GroupMap::const_iterator group_it = group_map_.find(group_name);
 579   if (group_it == group_map_.end()) {
 580     // TODO(mmenke):  This is actually reached in the wild, for unknown reasons.
 581     // Would be great to understand why, and if it's a bug, fix it.  If not,
 582     // should have a test for that case.
 583     NOTREACHED();
 584     return LOAD_STATE_IDLE;
 585   }
 586
 587   const Group& group = *group_it->second;
 588   if (group.HasConnectJobForHandle(handle)) {
 589     // Just return the state of the oldest ConnectJob.
 590     return (*group.jobs().begin())->GetLoadState();
 591   }
 592
 593   if (group.CanUseAdditionalSocketSlot(max_sockets_per_group_))
 594     return LOAD_STATE_WAITING_FOR_STALLED_SOCKET_POOL;
 595   return LOAD_STATE_WAITING_FOR_AVAILABLE_SOCKET;
 596 }
 597
 598 scoped_ptr<base::DictionaryValue> ClientSocketPoolBaseHelper::GetInfoAsValue(
 599     const std::string& name, const std::string& type) const {
 600   scoped_ptr<base::DictionaryValue> dict(new base::DictionaryValue());
 601   dict->SetString("name", name);
 602   dict->SetString("type", type);
 603   dict->SetInteger("handed_out_socket_count", handed_out_socket_count_);
 604   dict->SetInteger("connecting_socket_count", connecting_socket_count_);
 605   dict->SetInteger("idle_socket_count", idle_socket_count_);
 606   dict->SetInteger("max_socket_count", max_sockets_);
 607   dict->SetInteger("max_sockets_per_group", max_sockets_per_group_);
 608   dict->SetInteger("pool_generation_number", pool_generation_number_);
 609
 610   if (group_map_.empty())
 611     return dict.Pass();
 612
 613   base::DictionaryValue* all_groups_dict = new base::DictionaryValue();
 614   for (GroupMap::const_iterator it = group_map_.begin();
 615        it != group_map_.end(); it++) {
 616     const Group* group = it->second;
 617     base::DictionaryValue* group_dict = new base::DictionaryValue();
 618
 619     group_dict->SetInteger("pending_request_count",
 620                            group->pending_request_count());
 621     if (group->has_pending_requests()) {
 622       group_dict->SetString(
 623           "top_pending_priority",
 624           RequestPriorityToString(group->TopPendingPriority()));
 625     }
 626
 627     group_dict->SetInteger("active_socket_count", group->active_socket_count());
 628
 629     base::ListValue* idle_socket_list = new base::ListValue();
 630     std::list<IdleSocket>::const_iterator idle_socket;
 631     for (idle_socket = group->idle_sockets().begin();
 632          idle_socket != group->idle_sockets().end();
 633          idle_socket++) {
 634       int source_id = idle_socket->socket->NetLog().source().id;
 635       idle_socket_list->Append(new base::FundamentalValue(source_id));
 636     }
 637     group_dict->Set("idle_sockets", idle_socket_list);
 638
 639     base::ListValue* connect_jobs_list = new base::ListValue();
 640     std::list<ConnectJob*>::const_iterator job = group->jobs().begin();
 641     for (job = group->jobs().begin(); job != group->jobs().end(); job++) {
 642       int source_id = (*job)->net_log().source().id;
 643       connect_jobs_list->Append(new base::FundamentalValue(source_id));
 644     }
 645     group_dict->Set("connect_jobs", connect_jobs_list);
 646
 647     group_dict->SetBoolean("is_stalled", group->CanUseAdditionalSocketSlot(
 648                                              max_sockets_per_group_));
 649     group_dict->SetBoolean("backup_job_timer_is_running",
 650                            group->BackupJobTimerIsRunning());
 651
 652     all_groups_dict->SetWithoutPathExpansion(it->first, group_dict);
 653   }
 654   dict->Set("groups", all_groups_dict);
 655   return dict.Pass();
 656 }
 657
 658 bool ClientSocketPoolBaseHelper::IdleSocket::IsUsable() const {
 659   if (socket->WasEverUsed())
 660     return socket->IsConnectedAndIdle();
 661   return socket->IsConnected();
 662 }
 663
 664 bool ClientSocketPoolBaseHelper::IdleSocket::ShouldCleanup(
 665     base::TimeTicks now,
 666     base::TimeDelta timeout) const {
 667   bool timed_out = (now - start_time) >= timeout;
 668   if (timed_out)
 669     return true;
 670   return !IsUsable();
 671 }
 672
 673 void ClientSocketPoolBaseHelper::CleanupIdleSockets(bool force) {
 674   if (idle_socket_count_ == 0)
 675     return;
 676
 677   // Current time value. Retrieving it once at the function start rather than
 678   // inside the inner loop, since it shouldn't change by any meaningful amount.
 679   base::TimeTicks now = base::TimeTicks::Now();
 680
 681   GroupMap::iterator i = group_map_.begin();
 682   while (i != group_map_.end()) {
 683     Group* group = i->second;
 684
 685     std::list<IdleSocket>::iterator j = group->mutable_idle_sockets()->begin();
 686     while (j != group->idle_sockets().end()) {
 687       base::TimeDelta timeout =
 688           j->socket->WasEverUsed() ?
 689           used_idle_socket_timeout_ : unused_idle_socket_timeout_;
 690       if (force || j->ShouldCleanup(now, timeout)) {
 691         delete j->socket;
 692         j = group->mutable_idle_sockets()->erase(j);
 693         DecrementIdleCount();
 694       } else {
 695         ++j;
 696       }
 697     }
 698
 699     // Delete group if no longer needed.
 700     if (group->IsEmpty()) {
 701       RemoveGroup(i++);
 702     } else {
 703       ++i;
 704     }
 705   }
 706 }
 707
 708 ClientSocketPoolBaseHelper::Group* ClientSocketPoolBaseHelper::GetOrCreateGroup(
 709     const std::string& group_name) {
 710   GroupMap::iterator it = group_map_.find(group_name);
 711   if (it != group_map_.end())
 712     return it->second;
 713   Group* group = new Group;
 714   group_map_[group_name] = group;
 715   return group;
 716 }
 717
 718 void ClientSocketPoolBaseHelper::RemoveGroup(const std::string& group_name) {
 719   GroupMap::iterator it = group_map_.find(group_name);
 720   CHECK(it != group_map_.end());
 721
 722   RemoveGroup(it);
 723 }
 724
 725 void ClientSocketPoolBaseHelper::RemoveGroup(GroupMap::iterator it) {
 726   delete it->second;
 727   group_map_.erase(it);
 728 }
 729
 730 // static
 731 bool ClientSocketPoolBaseHelper::connect_backup_jobs_enabled() {
 732   return g_connect_backup_jobs_enabled;
 733 }
 734
 735 // static
 736 bool ClientSocketPoolBaseHelper::set_connect_backup_jobs_enabled(bool enabled) {
 737   bool old_value = g_connect_backup_jobs_enabled;
 738   g_connect_backup_jobs_enabled = enabled;
 739   return old_value;
 740 }
 741
 742 void ClientSocketPoolBaseHelper::EnableConnectBackupJobs() {
 743   connect_backup_jobs_enabled_ = g_connect_backup_jobs_enabled;
 744 }
 745
 746 void ClientSocketPoolBaseHelper::IncrementIdleCount() {
 747   if (++idle_socket_count_ == 1 && use_cleanup_timer_)
 748     StartIdleSocketTimer();
 749 }
 750
 751 void ClientSocketPoolBaseHelper::DecrementIdleCount() {
 752   if (--idle_socket_count_ == 0)
 753     timer_.Stop();
 754 }
 755
 756 // static
 757 bool ClientSocketPoolBaseHelper::cleanup_timer_enabled() {
 758   return g_cleanup_timer_enabled;
 759 }
 760
 761 // static
 762 bool ClientSocketPoolBaseHelper::set_cleanup_timer_enabled(bool enabled) {
 763   bool old_value = g_cleanup_timer_enabled;
 764   g_cleanup_timer_enabled = enabled;
 765   return old_value;
 766 }
 767
 768 void ClientSocketPoolBaseHelper::StartIdleSocketTimer() {
 769   timer_.Start(FROM_HERE, TimeDelta::FromSeconds(kCleanupInterval), this,
 770                &ClientSocketPoolBaseHelper::OnCleanupTimerFired);
 771 }
 772
 773 void ClientSocketPoolBaseHelper::ReleaseSocket(const std::string& group_name,
 774                                                scoped_ptr<StreamSocket> socket,
 775                                                int id) {
 776   GroupMap::iterator i = group_map_.find(group_name);
 777   CHECK(i != group_map_.end());
 778
 779   Group* group = i->second;
 780
 781   CHECK_GT(handed_out_socket_count_, 0);
 782   handed_out_socket_count_--;
 783
 784   CHECK_GT(group->active_socket_count(), 0);
 785   group->DecrementActiveSocketCount();
 786
 787   const bool can_reuse = socket->IsConnectedAndIdle() &&
 788       id == pool_generation_number_;
 789   if (can_reuse) {
 790     // Add it to the idle list.
 791     AddIdleSocket(socket.Pass(), group);
 792     OnAvailableSocketSlot(group_name, group);
 793   } else {
 794     socket.reset();
 795   }
 796
 797   CheckForStalledSocketGroups();
 798 }
 799
 800 void ClientSocketPoolBaseHelper::CheckForStalledSocketGroups() {
 801   // If we have idle sockets, see if we can give one to the top-stalled group.
 802   std::string top_group_name;
 803   Group* top_group = NULL;
 804   if (!FindTopStalledGroup(&top_group, &top_group_name)) {
 805     // There may still be a stalled group in a lower level pool.
 806     for (std::set<LowerLayeredPool*>::iterator it = lower_pools_.begin();
 807          it != lower_pools_.end();
 808          ++it) {
 809        if ((*it)->IsStalled()) {
 810          CloseOneIdleSocket();
 811          break;
 812        }
 813     }
 814     return;
 815   }
 816
 817   if (ReachedMaxSocketsLimit()) {
 818     if (idle_socket_count() > 0) {
 819       CloseOneIdleSocket();
 820     } else {
 821       // We can't activate more sockets since we're already at our global
 822       // limit.
 823       return;
 824     }
 825   }
 826
 827   // Note:  we don't loop on waking stalled groups.  If the stalled group is at
 828   //        its limit, may be left with other stalled groups that could be
 829   //        woken.  This isn't optimal, but there is no starvation, so to avoid
 830   //        the looping we leave it at this.
 831   OnAvailableSocketSlot(top_group_name, top_group);
 832 }
 833
 834 // Search for the highest priority pending request, amongst the groups that
 835 // are not at the |max_sockets_per_group_| limit. Note: for requests with
 836 // the same priority, the winner is based on group hash ordering (and not
 837 // insertion order).
 838 bool ClientSocketPoolBaseHelper::FindTopStalledGroup(
 839     Group** group,
 840     std::string* group_name) const {
 841   CHECK((group && group_name) || (!group && !group_name));
 842   Group* top_group = NULL;
 843   const std::string* top_group_name = NULL;
 844   bool has_stalled_group = false;
 845   for (GroupMap::const_iterator i = group_map_.begin();
 846        i != group_map_.end(); ++i) {
 847     Group* curr_group = i->second;
 848     if (!curr_group->has_pending_requests())
 849       continue;
 850     if (curr_group->CanUseAdditionalSocketSlot(max_sockets_per_group_)) {
 851       if (!group)
 852         return true;
 853       has_stalled_group = true;
 854       bool has_higher_priority = !top_group ||
 855           curr_group->TopPendingPriority() > top_group->TopPendingPriority();
 856       if (has_higher_priority) {
 857         top_group = curr_group;
 858         top_group_name = &i->first;
 859       }
 860     }
 861   }
 862
 863   if (top_group) {
 864     CHECK(group);
 865     *group = top_group;
 866     *group_name = *top_group_name;
 867   } else {
 868     CHECK(!has_stalled_group);
 869   }
 870   return has_stalled_group;
 871 }
 872
 873 void ClientSocketPoolBaseHelper::OnConnectJobComplete(
 874     int result, ConnectJob* job) {
 875   DCHECK_NE(ERR_IO_PENDING, result);
 876   const std::string group_name = job->group_name();
 877   GroupMap::iterator group_it = group_map_.find(group_name);
 878   CHECK(group_it != group_map_.end());
 879   Group* group = group_it->second;
 880
 881   scoped_ptr<StreamSocket> socket = job->PassSocket();
 882
 883   // Copies of these are needed because |job| may be deleted before they are
 884   // accessed.
 885   BoundNetLog job_log = job->net_log();
 886   LoadTimingInfo::ConnectTiming connect_timing = job->connect_timing();
 887
 888   // RemoveConnectJob(job, _) must be called by all branches below;
 889   // otherwise, |job| will be leaked.
 890
 891   if (result == OK) {
 892     DCHECK(socket.get());
 893     RemoveConnectJob(job, group);
 894     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 895     if (request) {
 896       LogBoundConnectJobToRequest(job_log.source(), *request);
 897       HandOutSocket(
 898           socket.Pass(), ClientSocketHandle::UNUSED, connect_timing,
 899           request->handle(), base::TimeDelta(), group, request->net_log());
 900       request->net_log().EndEvent(NetLog::TYPE_SOCKET_POOL);
 901       InvokeUserCallbackLater(request->handle(), request->callback(), result);
 902     } else {
 903       AddIdleSocket(socket.Pass(), group);
 904       OnAvailableSocketSlot(group_name, group);
 905       CheckForStalledSocketGroups();
 906     }
 907   } else {
 908     // If we got a socket, it must contain error information so pass that
 909     // up so that the caller can retrieve it.
 910     bool handed_out_socket = false;
 911     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 912     if (request) {
 913       LogBoundConnectJobToRequest(job_log.source(), *request);
 914       job->GetAdditionalErrorState(request->handle());
 915       RemoveConnectJob(job, group);
 916       if (socket.get()) {
 917         handed_out_socket = true;
 918         HandOutSocket(socket.Pass(), ClientSocketHandle::UNUSED,
 919                       connect_timing, request->handle(), base::TimeDelta(),
 920                       group, request->net_log());
 921       }
 922       request->net_log().EndEventWithNetErrorCode(
 923           NetLog::TYPE_SOCKET_POOL, result);
 924       InvokeUserCallbackLater(request->handle(), request->callback(), result);
 925     } else {
 926       RemoveConnectJob(job, group);
 927     }
 928     if (!handed_out_socket) {
 929       OnAvailableSocketSlot(group_name, group);
 930       CheckForStalledSocketGroups();
 931     }
 932   }
 933 }
 934
 935 void ClientSocketPoolBaseHelper::OnIPAddressChanged() {
 936   FlushWithError(ERR_NETWORK_CHANGED);
 937 }
 938
 939 void ClientSocketPoolBaseHelper::FlushWithError(int error) {
 940   pool_generation_number_++;
 941   CancelAllConnectJobs();
 942   CloseIdleSockets();
 943   CancelAllRequestsWithError(error);
 944 }
 945
 946 void ClientSocketPoolBaseHelper::RemoveConnectJob(ConnectJob* job,
 947                                                   Group* group) {
 948   CHECK_GT(connecting_socket_count_, 0);
 949   connecting_socket_count_--;
 950
 951   DCHECK(group);
 952   group->RemoveJob(job);
 953 }
 954
 955 void ClientSocketPoolBaseHelper::OnAvailableSocketSlot(
 956     const std::string& group_name, Group* group) {
 957   DCHECK(ContainsKey(group_map_, group_name));
 958   if (group->IsEmpty()) {
 959     RemoveGroup(group_name);
 960   } else if (group->has_pending_requests()) {
 961     ProcessPendingRequest(group_name, group);
 962   }
 963 }
 964
 965 void ClientSocketPoolBaseHelper::ProcessPendingRequest(
 966     const std::string& group_name, Group* group) {
 967   const Request* next_request = group->GetNextPendingRequest();
 968   DCHECK(next_request);
 969
 970   // If the group has no idle sockets, and can't make use of an additional slot,
 971   // either because it's at the limit or because it's at the socket per group
 972   // limit, then there's nothing to do.
 973   if (group->idle_sockets().empty() &&
 974       !group->CanUseAdditionalSocketSlot(max_sockets_per_group_)) {
 975     return;
 976   }
 977
 978   int rv = RequestSocketInternal(group_name, *next_request);
 979   if (rv != ERR_IO_PENDING) {
 980     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 981     DCHECK(request);
 982     if (group->IsEmpty())
 983       RemoveGroup(group_name);
 984
 985     request->net_log().EndEventWithNetErrorCode(NetLog::TYPE_SOCKET_POOL, rv);
 986     InvokeUserCallbackLater(request->handle(), request->callback(), rv);
 987   }
 988 }
 989
 990 void ClientSocketPoolBaseHelper::HandOutSocket(
 991     scoped_ptr<StreamSocket> socket,
 992     ClientSocketHandle::SocketReuseType reuse_type,
 993     const LoadTimingInfo::ConnectTiming& connect_timing,
 994     ClientSocketHandle* handle,
 995     base::TimeDelta idle_time,
 996     Group* group,
 997     const BoundNetLog& net_log) {
 998   DCHECK(socket);
 999   handle->SetSocket(socket.Pass());
1000   handle->set_reuse_type(reuse_type);
1001   handle->set_idle_time(idle_time);
1002   handle->set_pool_id(pool_generation_number_);
1003   handle->set_connect_timing(connect_timing);
1004
1005   if (handle->is_reused()) {
1006     net_log.AddEvent(
1007         NetLog::TYPE_SOCKET_POOL_REUSED_AN_EXISTING_SOCKET,
1008         NetLog::IntegerCallback(
1009             "idle_ms", static_cast<int>(idle_time.InMilliseconds())));
1010   }
1011
1012   net_log.AddEvent(
1013       NetLog::TYPE_SOCKET_POOL_BOUND_TO_SOCKET,
1014       handle->socket()->NetLog().source().ToEventParametersCallback());
1015
1016   handed_out_socket_count_++;
1017   group->IncrementActiveSocketCount();
1018 }
1019
1020 void ClientSocketPoolBaseHelper::AddIdleSocket(
1021     scoped_ptr<StreamSocket> socket,
1022     Group* group) {
1023   DCHECK(socket);
1024   IdleSocket idle_socket;
1025   idle_socket.socket = socket.release();
1026   idle_socket.start_time = base::TimeTicks::Now();
1027
1028   group->mutable_idle_sockets()->push_back(idle_socket);
1029   IncrementIdleCount();
1030 }
1031
1032 void ClientSocketPoolBaseHelper::CancelAllConnectJobs() {
1033   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end();) {
1034     Group* group = i->second;
1035     connecting_socket_count_ -= group->jobs().size();
1036     group->RemoveAllJobs();
1037
1038     // Delete group if no longer needed.
1039     if (group->IsEmpty()) {
1040       // RemoveGroup() will call .erase() which will invalidate the iterator,
1041       // but i will already have been incremented to a valid iterator before
1042       // RemoveGroup() is called.
1043       RemoveGroup(i++);
1044     } else {
1045       ++i;
1046     }
1047   }
1048   DCHECK_EQ(0, connecting_socket_count_);
1049 }
1050
1051 void ClientSocketPoolBaseHelper::CancelAllRequestsWithError(int error) {
1052   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end();) {
1053     Group* group = i->second;
1054
1055     while (true) {
1056       scoped_ptr<const Request> request = group->PopNextPendingRequest();
1057       if (!request)
1058         break;
1059       InvokeUserCallbackLater(request->handle(), request->callback(), error);
1060     }
1061
1062     // Delete group if no longer needed.
1063     if (group->IsEmpty()) {
1064       // RemoveGroup() will call .erase() which will invalidate the iterator,
1065       // but i will already have been incremented to a valid iterator before
1066       // RemoveGroup() is called.
1067       RemoveGroup(i++);
1068     } else {
1069       ++i;
1070     }
1071   }
1072 }
1073
1074 bool ClientSocketPoolBaseHelper::ReachedMaxSocketsLimit() const {
1075   // Each connecting socket will eventually connect and be handed out.
1076   int total = handed_out_socket_count_ + connecting_socket_count_ +
1077       idle_socket_count();
1078   // There can be more sockets than the limit since some requests can ignore
1079   // the limit
1080   if (total < max_sockets_)
1081     return false;
1082   return true;
1083 }
1084
1085 bool ClientSocketPoolBaseHelper::CloseOneIdleSocket() {
1086   if (idle_socket_count() == 0)
1087     return false;
1088   return CloseOneIdleSocketExceptInGroup(NULL);
1089 }
1090
1091 bool ClientSocketPoolBaseHelper::CloseOneIdleSocketExceptInGroup(
1092     const Group* exception_group) {
1093   CHECK_GT(idle_socket_count(), 0);
1094
1095   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end(); ++i) {
1096     Group* group = i->second;
1097     if (exception_group == group)
1098       continue;
1099     std::list<IdleSocket>* idle_sockets = group->mutable_idle_sockets();
1100
1101     if (!idle_sockets->empty()) {
1102       delete idle_sockets->front().socket;
1103       idle_sockets->pop_front();
1104       DecrementIdleCount();
1105       if (group->IsEmpty())
1106         RemoveGroup(i);
1107
1108       return true;
1109     }
1110   }
1111
1112   return false;
1113 }
1114
1115 bool ClientSocketPoolBaseHelper::CloseOneIdleConnectionInHigherLayeredPool() {
1116   // This pool doesn't have any idle sockets. It's possible that a pool at a
1117   // higher layer is holding one of this sockets active, but it's actually idle.
1118   // Query the higher layers.
1119   for (std::set<HigherLayeredPool*>::const_iterator it = higher_pools_.begin();
1120        it != higher_pools_.end(); ++it) {
1121     if ((*it)->CloseOneIdleConnection())
1122       return true;
1123   }
1124   return false;
1125 }
1126
1127 void ClientSocketPoolBaseHelper::InvokeUserCallbackLater(
1128     ClientSocketHandle* handle, const CompletionCallback& callback, int rv) {
1129   CHECK(!ContainsKey(pending_callback_map_, handle));
1130   pending_callback_map_[handle] = CallbackResultPair(callback, rv);
1131   base::MessageLoop::current()->PostTask(
1132       FROM_HERE,
1133       base::Bind(&ClientSocketPoolBaseHelper::InvokeUserCallback,
1134                  weak_factory_.GetWeakPtr(), handle));
1135 }
1136
1137 void ClientSocketPoolBaseHelper::InvokeUserCallback(
1138     ClientSocketHandle* handle) {
1139   PendingCallbackMap::iterator it = pending_callback_map_.find(handle);
1140
1141   // Exit if the request has already been cancelled.
1142   if (it == pending_callback_map_.end())
1143     return;
1144
1145   CHECK(!handle->is_initialized());
1146   CompletionCallback callback = it->second.callback;
1147   int result = it->second.result;
1148   pending_callback_map_.erase(it);
1149   callback.Run(result);
1150 }
1151
1152 void ClientSocketPoolBaseHelper::TryToCloseSocketsInLayeredPools() {
1153   while (IsStalled()) {
1154     // Closing a socket will result in calling back into |this| to use the freed
1155     // socket slot, so nothing else is needed.
1156     if (!CloseOneIdleConnectionInHigherLayeredPool())
1157       return;
1158   }
1159 }
1160
1161 ClientSocketPoolBaseHelper::Group::Group()
1162     : unassigned_job_count_(0),
1163       pending_requests_(NUM_PRIORITIES),
1164       active_socket_count_(0) {}
1165
1166 ClientSocketPoolBaseHelper::Group::~Group() {
1167   DCHECK_EQ(0u, unassigned_job_count_);
1168 }
1169
1170 void ClientSocketPoolBaseHelper::Group::StartBackupJobTimer(
1171     const std::string& group_name,
1172     ClientSocketPoolBaseHelper* pool) {
1173   // Only allow one timer to run at a time.
1174   if (BackupJobTimerIsRunning())
1175     return;
1176
1177   // Unretained here is okay because |backup_job_timer_| is
1178   // automatically cancelled when it's destroyed.
1179   backup_job_timer_.Start(
1180       FROM_HERE, pool->ConnectRetryInterval(),
1181       base::Bind(&Group::OnBackupJobTimerFired, base::Unretained(this),
1182                  group_name, pool));
1183 }
1184
1185 bool ClientSocketPoolBaseHelper::Group::BackupJobTimerIsRunning() const {
1186   return backup_job_timer_.IsRunning();
1187 }
1188
1189 bool ClientSocketPoolBaseHelper::Group::TryToUseUnassignedConnectJob() {
1190   SanityCheck();
1191
1192   if (unassigned_job_count_ == 0)
1193     return false;
1194   --unassigned_job_count_;
1195   return true;
1196 }
1197
1198 void ClientSocketPoolBaseHelper::Group::AddJob(scoped_ptr<ConnectJob> job,
1199                                                bool is_preconnect) {
1200   SanityCheck();
1201
1202   if (is_preconnect)
1203     ++unassigned_job_count_;
1204   jobs_.push_back(job.release());
1205 }
1206
1207 void ClientSocketPoolBaseHelper::Group::RemoveJob(ConnectJob* job) {
1208   scoped_ptr<ConnectJob> owned_job(job);
1209   SanityCheck();
1210
1211   // Check that |job| is in the list.
1212   DCHECK_EQ(*std::find(jobs_.begin(), jobs_.end(), job), job);
1213   jobs_.remove(job);
1214   size_t job_count = jobs_.size();
1215   if (job_count < unassigned_job_count_)
1216     unassigned_job_count_ = job_count;
1217
1218   // If we've got no more jobs for this group, then we no longer need a
1219   // backup job either.
1220   if (jobs_.empty())
1221     backup_job_timer_.Stop();
1222 }
1223
1224 void ClientSocketPoolBaseHelper::Group::OnBackupJobTimerFired(
1225     std::string group_name,
1226     ClientSocketPoolBaseHelper* pool) {
1227   // If there are no more jobs pending, there is no work to do.
1228   // If we've done our cleanups correctly, this should not happen.
1229   if (jobs_.empty()) {
1230     NOTREACHED();
1231     return;
1232   }
1233
1234   // If our old job is waiting on DNS, or if we can't create any sockets
1235   // right now due to limits, just reset the timer.
1236   if (pool->ReachedMaxSocketsLimit() ||
1237       !HasAvailableSocketSlot(pool->max_sockets_per_group_) ||
1238       (*jobs_.begin())->GetLoadState() == LOAD_STATE_RESOLVING_HOST) {
1239     StartBackupJobTimer(group_name, pool);
1240     return;
1241   }
1242
1243   if (pending_requests_.empty())
1244     return;
1245
1246   scoped_ptr<ConnectJob> backup_job =
1247       pool->connect_job_factory_->NewConnectJob(
1248           group_name, *pending_requests_.FirstMax().value(), pool);
1249   backup_job->net_log().AddEvent(NetLog::TYPE_BACKUP_CONNECT_JOB_CREATED);
1250   int rv = backup_job->Connect();
1251   pool->connecting_socket_count_++;
1252   ConnectJob* raw_backup_job = backup_job.get();
1253   AddJob(backup_job.Pass(), false);
1254   if (rv != ERR_IO_PENDING)
1255     pool->OnConnectJobComplete(rv, raw_backup_job);
1256 }
1257
1258 void ClientSocketPoolBaseHelper::Group::SanityCheck() {
1259   DCHECK_LE(unassigned_job_count_, jobs_.size());
1260 }
1261
1262 void ClientSocketPoolBaseHelper::Group::RemoveAllJobs() {
1263   SanityCheck();
1264
1265   // Delete active jobs.
1266   STLDeleteElements(&jobs_);
1267   unassigned_job_count_ = 0;
1268
1269   // Stop backup job timer.
1270   backup_job_timer_.Stop();
1271 }
1272
1273 const ClientSocketPoolBaseHelper::Request*
1274 ClientSocketPoolBaseHelper::Group::GetNextPendingRequest() const {
1275   return
1276       pending_requests_.empty() ? NULL : pending_requests_.FirstMax().value();
1277 }
1278
1279 bool ClientSocketPoolBaseHelper::Group::HasConnectJobForHandle(
1280     const ClientSocketHandle* handle) const {
1281   // Search the first |jobs_.size()| pending requests for |handle|.
1282   // If it's farther back in the deque than that, it doesn't have a
1283   // corresponding ConnectJob.
1284   size_t i = 0;
1285   for (RequestQueue::Pointer pointer = pending_requests_.FirstMax();
1286        !pointer.is_null() && i < jobs_.size();
1287        pointer = pending_requests_.GetNextTowardsLastMin(pointer), ++i) {
1288     if (pointer.value()->handle() == handle)
1289       return true;
1290   }
1291   return false;
1292 }
1293
1294 void ClientSocketPoolBaseHelper::Group::InsertPendingRequest(
1295     scoped_ptr<const Request> request) {
1296   // This value must be cached before we release |request|.
1297   RequestPriority priority = request->priority();
1298   if (request->ignore_limits()) {
1299     // Put requests with ignore_limits == true (which should have
1300     // priority == MAXIMUM_PRIORITY) ahead of other requests with
1301     // MAXIMUM_PRIORITY.
1302     DCHECK_EQ(priority, MAXIMUM_PRIORITY);
1303     pending_requests_.InsertAtFront(request.release(), priority);
1304   } else {
1305     pending_requests_.Insert(request.release(), priority);
1306   }
1307 }
1308
1309 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1310 ClientSocketPoolBaseHelper::Group::PopNextPendingRequest() {
1311   if (pending_requests_.empty())
1312     return scoped_ptr<const ClientSocketPoolBaseHelper::Request>();
1313   return RemovePendingRequest(pending_requests_.FirstMax());
1314 }
1315
1316 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1317 ClientSocketPoolBaseHelper::Group::FindAndRemovePendingRequest(
1318     ClientSocketHandle* handle) {
1319   for (RequestQueue::Pointer pointer = pending_requests_.FirstMax();
1320        !pointer.is_null();
1321        pointer = pending_requests_.GetNextTowardsLastMin(pointer)) {
1322     if (pointer.value()->handle() == handle) {
1323       scoped_ptr<const Request> request = RemovePendingRequest(pointer);
1324       return request.Pass();
1325     }
1326   }
1327   return scoped_ptr<const ClientSocketPoolBaseHelper::Request>();
1328 }
1329
1330 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1331 ClientSocketPoolBaseHelper::Group::RemovePendingRequest(
1332     const RequestQueue::Pointer& pointer) {
1333   // TODO(eroman): Temporary for debugging http://crbug.com/467797.
1334   CHECK(!pointer.is_null());
1335   scoped_ptr<const Request> request(pointer.value());
1336   pending_requests_.Erase(pointer);
1337   // If there are no more requests, kill the backup timer.
1338   if (pending_requests_.empty())
1339     backup_job_timer_.Stop();
1340   request->CrashIfInvalid();
1341   return request.Pass();
1342 }
1343
1344 }  // namespace internal
1345
1346 }  // namespace net