net/socket/client_socket_pool_base.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/socket/client_socket_pool_base.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/compiler_specific.h"
  10 #include "base/format_macros.h"
  11 #include "base/location.h"
  12 #include "base/logging.h"
  13 #include "base/single_thread_task_runner.h"
  14 #include "base/stl_util.h"
  15 #include "base/strings/string_util.h"
  16 #include "base/thread_task_runner_handle.h"
  17 #include "base/time/time.h"
  18 #include "base/values.h"
  19 #include "net/base/net_errors.h"
  20 #include "net/log/net_log.h"
  21
  22 using base::TimeDelta;
  23
  24 namespace net {
  25
  26 namespace {
  27
  28 // Indicate whether we should enable idle socket cleanup timer. When timer is
  29 // disabled, sockets are closed next time a socket request is made.
  30 // Keep this enabled for windows as long as we support Windows XP, see the note
  31 // in kCleanupInterval below.
  32 #if defined(OS_WIN)
  33 bool g_cleanup_timer_enabled = true;
  34 #else
  35 bool g_cleanup_timer_enabled = false;
  36 #endif
  37
  38 // The timeout value, in seconds, used to clean up idle sockets that can't be
  39 // reused.
  40 //
  41 // Note: It's important to close idle sockets that have received data as soon
  42 // as possible because the received data may cause BSOD on Windows XP under
  43 // some conditions.  See http://crbug.com/4606.
  44 const int kCleanupInterval = 10;  // DO NOT INCREASE THIS TIMEOUT.
  45
  46 // Indicate whether or not we should establish a new transport layer connection
  47 // after a certain timeout has passed without receiving an ACK.
  48 bool g_connect_backup_jobs_enabled = true;
  49
  50 }  // namespace
  51
  52 ConnectJob::ConnectJob(const std::string& group_name,
  53                        base::TimeDelta timeout_duration,
  54                        RequestPriority priority,
  55                        Delegate* delegate,
  56                        const BoundNetLog& net_log)
  57     : group_name_(group_name),
  58       timeout_duration_(timeout_duration),
  59       priority_(priority),
  60       delegate_(delegate),
  61       net_log_(net_log),
  62       idle_(true) {
  63   DCHECK(!group_name.empty());
  64   DCHECK(delegate);
  65   net_log.BeginEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB,
  66                      NetLog::StringCallback("group_name", &group_name_));
  67 }
  68
  69 ConnectJob::~ConnectJob() {
  70   net_log().EndEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB);
  71 }
  72
  73 scoped_ptr<StreamSocket> ConnectJob::PassSocket() {
  74   return socket_.Pass();
  75 }
  76
  77 int ConnectJob::Connect() {
  78   if (timeout_duration_ != base::TimeDelta())
  79     timer_.Start(FROM_HERE, timeout_duration_, this, &ConnectJob::OnTimeout);
  80
  81   idle_ = false;
  82
  83   LogConnectStart();
  84
  85   int rv = ConnectInternal();
  86
  87   if (rv != ERR_IO_PENDING) {
  88     LogConnectCompletion(rv);
  89     delegate_ = NULL;
  90   }
  91
  92   return rv;
  93 }
  94
  95 void ConnectJob::SetSocket(scoped_ptr<StreamSocket> socket) {
  96   if (socket) {
  97     net_log().AddEvent(NetLog::TYPE_CONNECT_JOB_SET_SOCKET,
  98                        socket->NetLog().source().ToEventParametersCallback());
  99   }
 100   socket_ = socket.Pass();
 101 }
 102
 103 void ConnectJob::NotifyDelegateOfCompletion(int rv) {
 104   // The delegate will own |this|.
 105   Delegate* delegate = delegate_;
 106   delegate_ = NULL;
 107
 108   LogConnectCompletion(rv);
 109   delegate->OnConnectJobComplete(rv, this);
 110 }
 111
 112 void ConnectJob::ResetTimer(base::TimeDelta remaining_time) {
 113   timer_.Stop();
 114   timer_.Start(FROM_HERE, remaining_time, this, &ConnectJob::OnTimeout);
 115 }
 116
 117 void ConnectJob::LogConnectStart() {
 118   connect_timing_.connect_start = base::TimeTicks::Now();
 119   net_log().BeginEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_CONNECT);
 120 }
 121
 122 void ConnectJob::LogConnectCompletion(int net_error) {
 123   connect_timing_.connect_end = base::TimeTicks::Now();
 124   net_log().EndEventWithNetErrorCode(
 125       NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_CONNECT, net_error);
 126 }
 127
 128 void ConnectJob::OnTimeout() {
 129   // Make sure the socket is NULL before calling into |delegate|.
 130   SetSocket(scoped_ptr<StreamSocket>());
 131
 132   net_log_.AddEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_TIMED_OUT);
 133
 134   NotifyDelegateOfCompletion(ERR_TIMED_OUT);
 135 }
 136
 137 namespace internal {
 138
 139 ClientSocketPoolBaseHelper::Request::Request(
 140     ClientSocketHandle* handle,
 141     const CompletionCallback& callback,
 142     RequestPriority priority,
 143     bool ignore_limits,
 144     Flags flags,
 145     const BoundNetLog& net_log)
 146     : handle_(handle),
 147       callback_(callback),
 148       priority_(priority),
 149       ignore_limits_(ignore_limits),
 150       flags_(flags),
 151       net_log_(net_log) {
 152   if (ignore_limits_)
 153     DCHECK_EQ(priority_, MAXIMUM_PRIORITY);
 154 }
 155
 156 ClientSocketPoolBaseHelper::Request::~Request() {
 157   liveness_ = DEAD;
 158 }
 159
 160 void ClientSocketPoolBaseHelper::Request::CrashIfInvalid() const {
 161   CHECK_EQ(liveness_, ALIVE);
 162 }
 163
 164 ClientSocketPoolBaseHelper::ClientSocketPoolBaseHelper(
 165     HigherLayeredPool* pool,
 166     int max_sockets,
 167     int max_sockets_per_group,
 168     base::TimeDelta unused_idle_socket_timeout,
 169     base::TimeDelta used_idle_socket_timeout,
 170     ConnectJobFactory* connect_job_factory)
 171     : idle_socket_count_(0),
 172       connecting_socket_count_(0),
 173       handed_out_socket_count_(0),
 174       max_sockets_(max_sockets),
 175       max_sockets_per_group_(max_sockets_per_group),
 176       use_cleanup_timer_(g_cleanup_timer_enabled),
 177       unused_idle_socket_timeout_(unused_idle_socket_timeout),
 178       used_idle_socket_timeout_(used_idle_socket_timeout),
 179       connect_job_factory_(connect_job_factory),
 180       connect_backup_jobs_enabled_(false),
 181       pool_generation_number_(0),
 182       pool_(pool),
 183       weak_factory_(this) {
 184   DCHECK_LE(0, max_sockets_per_group);
 185   DCHECK_LE(max_sockets_per_group, max_sockets);
 186
 187   NetworkChangeNotifier::AddIPAddressObserver(this);
 188 }
 189
 190 ClientSocketPoolBaseHelper::~ClientSocketPoolBaseHelper() {
 191   // Clean up any idle sockets and pending connect jobs.  Assert that we have no
 192   // remaining active sockets or pending requests.  They should have all been
 193   // cleaned up prior to |this| being destroyed.
 194   FlushWithError(ERR_ABORTED);
 195   DCHECK(group_map_.empty());
 196   DCHECK(pending_callback_map_.empty());
 197   DCHECK_EQ(0, connecting_socket_count_);
 198   CHECK(higher_pools_.empty());
 199
 200   NetworkChangeNotifier::RemoveIPAddressObserver(this);
 201
 202   // Remove from lower layer pools.
 203   for (std::set<LowerLayeredPool*>::iterator it = lower_pools_.begin();
 204        it != lower_pools_.end();
 205        ++it) {
 206     (*it)->RemoveHigherLayeredPool(pool_);
 207   }
 208 }
 209
 210 ClientSocketPoolBaseHelper::CallbackResultPair::CallbackResultPair()
 211     : result(OK) {
 212 }
 213
 214 ClientSocketPoolBaseHelper::CallbackResultPair::CallbackResultPair(
 215     const CompletionCallback& callback_in, int result_in)
 216     : callback(callback_in),
 217       result(result_in) {
 218 }
 219
 220 ClientSocketPoolBaseHelper::CallbackResultPair::~CallbackResultPair() {}
 221
 222 bool ClientSocketPoolBaseHelper::IsStalled() const {
 223   // If a lower layer pool is stalled, consider |this| stalled as well.
 224   for (std::set<LowerLayeredPool*>::const_iterator it = lower_pools_.begin();
 225        it != lower_pools_.end();
 226        ++it) {
 227     if ((*it)->IsStalled())
 228       return true;
 229   }
 230
 231   // If fewer than |max_sockets_| are in use, then clearly |this| is not
 232   // stalled.
 233   if ((handed_out_socket_count_ + connecting_socket_count_) < max_sockets_)
 234     return false;
 235   // So in order to be stalled, |this| must be using at least |max_sockets_| AND
 236   // |this| must have a request that is actually stalled on the global socket
 237   // limit.  To find such a request, look for a group that has more requests
 238   // than jobs AND where the number of sockets is less than
 239   // |max_sockets_per_group_|.  (If the number of sockets is equal to
 240   // |max_sockets_per_group_|, then the request is stalled on the group limit,
 241   // which does not count.)
 242   for (GroupMap::const_iterator it = group_map_.begin();
 243        it != group_map_.end(); ++it) {
 244     if (it->second->CanUseAdditionalSocketSlot(max_sockets_per_group_))
 245       return true;
 246   }
 247   return false;
 248 }
 249
 250 void ClientSocketPoolBaseHelper::AddLowerLayeredPool(
 251     LowerLayeredPool* lower_pool) {
 252   DCHECK(pool_);
 253   CHECK(!ContainsKey(lower_pools_, lower_pool));
 254   lower_pools_.insert(lower_pool);
 255   lower_pool->AddHigherLayeredPool(pool_);
 256 }
 257
 258 void ClientSocketPoolBaseHelper::AddHigherLayeredPool(
 259     HigherLayeredPool* higher_pool) {
 260   CHECK(higher_pool);
 261   CHECK(!ContainsKey(higher_pools_, higher_pool));
 262   higher_pools_.insert(higher_pool);
 263 }
 264
 265 void ClientSocketPoolBaseHelper::RemoveHigherLayeredPool(
 266     HigherLayeredPool* higher_pool) {
 267   CHECK(higher_pool);
 268   CHECK(ContainsKey(higher_pools_, higher_pool));
 269   higher_pools_.erase(higher_pool);
 270 }
 271
 272 int ClientSocketPoolBaseHelper::RequestSocket(
 273     const std::string& group_name,
 274     scoped_ptr<const Request> request) {
 275   CHECK(!request->callback().is_null());
 276   CHECK(request->handle());
 277
 278   // Cleanup any timed-out idle sockets if no timer is used.
 279   if (!use_cleanup_timer_)
 280     CleanupIdleSockets(false);
 281
 282   request->net_log().BeginEvent(NetLog::TYPE_SOCKET_POOL);
 283   Group* group = GetOrCreateGroup(group_name);
 284
 285   int rv = RequestSocketInternal(group_name, *request);
 286   if (rv != ERR_IO_PENDING) {
 287     request->net_log().EndEventWithNetErrorCode(NetLog::TYPE_SOCKET_POOL, rv);
 288     CHECK(!request->handle()->is_initialized());
 289     request.reset();
 290   } else {
 291     group->InsertPendingRequest(request.Pass());
 292     // Have to do this asynchronously, as closing sockets in higher level pools
 293     // call back in to |this|, which will cause all sorts of fun and exciting
 294     // re-entrancy issues if the socket pool is doing something else at the
 295     // time.
 296     if (group->CanUseAdditionalSocketSlot(max_sockets_per_group_)) {
 297       base::ThreadTaskRunnerHandle::Get()->PostTask(
 298           FROM_HERE,
 299           base::Bind(
 300               &ClientSocketPoolBaseHelper::TryToCloseSocketsInLayeredPools,
 301               weak_factory_.GetWeakPtr()));
 302     }
 303   }
 304   return rv;
 305 }
 306
 307 void ClientSocketPoolBaseHelper::RequestSockets(
 308     const std::string& group_name,
 309     const Request& request,
 310     int num_sockets) {
 311   DCHECK(request.callback().is_null());
 312   DCHECK(!request.handle());
 313
 314   // Cleanup any timed out idle sockets if no timer is used.
 315   if (!use_cleanup_timer_)
 316     CleanupIdleSockets(false);
 317
 318   if (num_sockets > max_sockets_per_group_) {
 319     num_sockets = max_sockets_per_group_;
 320   }
 321
 322   request.net_log().BeginEvent(
 323       NetLog::TYPE_SOCKET_POOL_CONNECTING_N_SOCKETS,
 324       NetLog::IntegerCallback("num_sockets", num_sockets));
 325
 326   Group* group = GetOrCreateGroup(group_name);
 327
 328   // RequestSocketsInternal() may delete the group.
 329   bool deleted_group = false;
 330
 331   int rv = OK;
 332   for (int num_iterations_left = num_sockets;
 333        group->NumActiveSocketSlots() < num_sockets &&
 334        num_iterations_left > 0 ; num_iterations_left--) {
 335     rv = RequestSocketInternal(group_name, request);
 336     if (rv < 0 && rv != ERR_IO_PENDING) {
 337       // We're encountering a synchronous error.  Give up.
 338       if (!ContainsKey(group_map_, group_name))
 339         deleted_group = true;
 340       break;
 341     }
 342     if (!ContainsKey(group_map_, group_name)) {
 343       // Unexpected.  The group should only be getting deleted on synchronous
 344       // error.
 345       NOTREACHED();
 346       deleted_group = true;
 347       break;
 348     }
 349   }
 350
 351   if (!deleted_group && group->IsEmpty())
 352     RemoveGroup(group_name);
 353
 354   if (rv == ERR_IO_PENDING)
 355     rv = OK;
 356   request.net_log().EndEventWithNetErrorCode(
 357       NetLog::TYPE_SOCKET_POOL_CONNECTING_N_SOCKETS, rv);
 358 }
 359
 360 int ClientSocketPoolBaseHelper::RequestSocketInternal(
 361     const std::string& group_name,
 362     const Request& request) {
 363   ClientSocketHandle* const handle = request.handle();
 364   const bool preconnecting = !handle;
 365   Group* group = GetOrCreateGroup(group_name);
 366
 367   if (!(request.flags() & NO_IDLE_SOCKETS)) {
 368     // Try to reuse a socket.
 369     if (AssignIdleSocketToRequest(request, group))
 370       return OK;
 371   }
 372
 373   // If there are more ConnectJobs than pending requests, don't need to do
 374   // anything.  Can just wait for the extra job to connect, and then assign it
 375   // to the request.
 376   if (!preconnecting && group->TryToUseUnassignedConnectJob())
 377     return ERR_IO_PENDING;
 378
 379   // Can we make another active socket now?
 380   if (!group->HasAvailableSocketSlot(max_sockets_per_group_) &&
 381       !request.ignore_limits()) {
 382     // TODO(willchan): Consider whether or not we need to close a socket in a
 383     // higher layered group. I don't think this makes sense since we would just
 384     // reuse that socket then if we needed one and wouldn't make it down to this
 385     // layer.
 386     request.net_log().AddEvent(
 387         NetLog::TYPE_SOCKET_POOL_STALLED_MAX_SOCKETS_PER_GROUP);
 388     return ERR_IO_PENDING;
 389   }
 390
 391   if (ReachedMaxSocketsLimit() && !request.ignore_limits()) {
 392     // NOTE(mmenke):  Wonder if we really need different code for each case
 393     // here.  Only reason for them now seems to be preconnects.
 394     if (idle_socket_count() > 0) {
 395       // There's an idle socket in this pool. Either that's because there's
 396       // still one in this group, but we got here due to preconnecting bypassing
 397       // idle sockets, or because there's an idle socket in another group.
 398       bool closed = CloseOneIdleSocketExceptInGroup(group);
 399       if (preconnecting && !closed)
 400         return ERR_PRECONNECT_MAX_SOCKET_LIMIT;
 401     } else {
 402       // We could check if we really have a stalled group here, but it requires
 403       // a scan of all groups, so just flip a flag here, and do the check later.
 404       request.net_log().AddEvent(NetLog::TYPE_SOCKET_POOL_STALLED_MAX_SOCKETS);
 405       return ERR_IO_PENDING;
 406     }
 407   }
 408
 409   // We couldn't find a socket to reuse, and there's space to allocate one,
 410   // so allocate and connect a new one.
 411   scoped_ptr<ConnectJob> connect_job(
 412       connect_job_factory_->NewConnectJob(group_name, request, this));
 413
 414   int rv = connect_job->Connect();
 415   if (rv == OK) {
 416     LogBoundConnectJobToRequest(connect_job->net_log().source(), request);
 417     if (!preconnecting) {
 418       HandOutSocket(connect_job->PassSocket(), ClientSocketHandle::UNUSED,
 419                     connect_job->connect_timing(), handle, base::TimeDelta(),
 420                     group, request.net_log());
 421     } else {
 422       AddIdleSocket(connect_job->PassSocket(), group);
 423     }
 424   } else if (rv == ERR_IO_PENDING) {
 425     // If we don't have any sockets in this group, set a timer for potentially
 426     // creating a new one.  If the SYN is lost, this backup socket may complete
 427     // before the slow socket, improving end user latency.
 428     if (connect_backup_jobs_enabled_ && group->IsEmpty()) {
 429       group->StartBackupJobTimer(group_name, this);
 430     }
 431
 432     connecting_socket_count_++;
 433
 434     group->AddJob(connect_job.Pass(), preconnecting);
 435   } else {
 436     LogBoundConnectJobToRequest(connect_job->net_log().source(), request);
 437     scoped_ptr<StreamSocket> error_socket;
 438     if (!preconnecting) {
 439       DCHECK(handle);
 440       connect_job->GetAdditionalErrorState(handle);
 441       error_socket = connect_job->PassSocket();
 442     }
 443     if (error_socket) {
 444       HandOutSocket(error_socket.Pass(), ClientSocketHandle::UNUSED,
 445                     connect_job->connect_timing(), handle, base::TimeDelta(),
 446                     group, request.net_log());
 447     } else if (group->IsEmpty()) {
 448       RemoveGroup(group_name);
 449     }
 450   }
 451
 452   return rv;
 453 }
 454
 455 bool ClientSocketPoolBaseHelper::AssignIdleSocketToRequest(
 456     const Request& request, Group* group) {
 457   std::list<IdleSocket>* idle_sockets = group->mutable_idle_sockets();
 458   std::list<IdleSocket>::iterator idle_socket_it = idle_sockets->end();
 459
 460   // Iterate through the idle sockets forwards (oldest to newest)
 461   //   * Delete any disconnected ones.
 462   //   * If we find a used idle socket, assign to |idle_socket|.  At the end,
 463   //   the |idle_socket_it| will be set to the newest used idle socket.
 464   for (std::list<IdleSocket>::iterator it = idle_sockets->begin();
 465        it != idle_sockets->end();) {
 466     if (!it->IsUsable()) {
 467       DecrementIdleCount();
 468       delete it->socket;
 469       it = idle_sockets->erase(it);
 470       continue;
 471     }
 472
 473     if (it->socket->WasEverUsed()) {
 474       // We found one we can reuse!
 475       idle_socket_it = it;
 476     }
 477
 478     ++it;
 479   }
 480
 481   // If we haven't found an idle socket, that means there are no used idle
 482   // sockets.  Pick the oldest (first) idle socket (FIFO).
 483
 484   if (idle_socket_it == idle_sockets->end() && !idle_sockets->empty())
 485     idle_socket_it = idle_sockets->begin();
 486
 487   if (idle_socket_it != idle_sockets->end()) {
 488     DecrementIdleCount();
 489     base::TimeDelta idle_time =
 490         base::TimeTicks::Now() - idle_socket_it->start_time;
 491     IdleSocket idle_socket = *idle_socket_it;
 492     idle_sockets->erase(idle_socket_it);
 493     // TODO(davidben): If |idle_time| is under some low watermark, consider
 494     // treating as UNUSED rather than UNUSED_IDLE. This will avoid
 495     // HttpNetworkTransaction retrying on some errors.
 496     ClientSocketHandle::SocketReuseType reuse_type =
 497         idle_socket.socket->WasEverUsed() ?
 498             ClientSocketHandle::REUSED_IDLE :
 499             ClientSocketHandle::UNUSED_IDLE;
 500
 501     // If this socket took multiple attempts to obtain, don't report those
 502     // every time it's reused, just to the first user.
 503     if (idle_socket.socket->WasEverUsed())
 504       idle_socket.socket->ClearConnectionAttempts();
 505
 506     HandOutSocket(
 507         scoped_ptr<StreamSocket>(idle_socket.socket),
 508         reuse_type,
 509         LoadTimingInfo::ConnectTiming(),
 510         request.handle(),
 511         idle_time,
 512         group,
 513         request.net_log());
 514     return true;
 515   }
 516
 517   return false;
 518 }
 519
 520 // static
 521 void ClientSocketPoolBaseHelper::LogBoundConnectJobToRequest(
 522     const NetLog::Source& connect_job_source, const Request& request) {
 523   request.net_log().AddEvent(NetLog::TYPE_SOCKET_POOL_BOUND_TO_CONNECT_JOB,
 524                              connect_job_source.ToEventParametersCallback());
 525 }
 526
 527 void ClientSocketPoolBaseHelper::CancelRequest(
 528     const std::string& group_name, ClientSocketHandle* handle) {
 529   PendingCallbackMap::iterator callback_it = pending_callback_map_.find(handle);
 530   if (callback_it != pending_callback_map_.end()) {
 531     int result = callback_it->second.result;
 532     pending_callback_map_.erase(callback_it);
 533     scoped_ptr<StreamSocket> socket = handle->PassSocket();
 534     if (socket) {
 535       if (result != OK)
 536         socket->Disconnect();
 537       ReleaseSocket(handle->group_name(), socket.Pass(), handle->id());
 538     }
 539     return;
 540   }
 541
 542   CHECK(ContainsKey(group_map_, group_name));
 543
 544   Group* group = GetOrCreateGroup(group_name);
 545
 546   // Search pending_requests for matching handle.
 547   scoped_ptr<const Request> request =
 548       group->FindAndRemovePendingRequest(handle);
 549   if (request) {
 550     request->net_log().AddEvent(NetLog::TYPE_CANCELLED);
 551     request->net_log().EndEvent(NetLog::TYPE_SOCKET_POOL);
 552
 553     // We let the job run, unless we're at the socket limit and there is
 554     // not another request waiting on the job.
 555     if (group->jobs().size() > group->pending_request_count() &&
 556         ReachedMaxSocketsLimit()) {
 557       RemoveConnectJob(*group->jobs().begin(), group);
 558       CheckForStalledSocketGroups();
 559     }
 560   }
 561 }
 562
 563 bool ClientSocketPoolBaseHelper::HasGroup(const std::string& group_name) const {
 564   return ContainsKey(group_map_, group_name);
 565 }
 566
 567 void ClientSocketPoolBaseHelper::CloseIdleSockets() {
 568   CleanupIdleSockets(true);
 569   DCHECK_EQ(0, idle_socket_count_);
 570 }
 571
 572 int ClientSocketPoolBaseHelper::IdleSocketCountInGroup(
 573     const std::string& group_name) const {
 574   GroupMap::const_iterator i = group_map_.find(group_name);
 575   CHECK(i != group_map_.end());
 576
 577   return i->second->idle_sockets().size();
 578 }
 579
 580 LoadState ClientSocketPoolBaseHelper::GetLoadState(
 581     const std::string& group_name,
 582     const ClientSocketHandle* handle) const {
 583   if (ContainsKey(pending_callback_map_, handle))
 584     return LOAD_STATE_CONNECTING;
 585
 586   GroupMap::const_iterator group_it = group_map_.find(group_name);
 587   if (group_it == group_map_.end()) {
 588     // TODO(mmenke):  This is actually reached in the wild, for unknown reasons.
 589     // Would be great to understand why, and if it's a bug, fix it.  If not,
 590     // should have a test for that case.
 591     NOTREACHED();
 592     return LOAD_STATE_IDLE;
 593   }
 594
 595   const Group& group = *group_it->second;
 596   if (group.HasConnectJobForHandle(handle)) {
 597     // Just return the state of the oldest ConnectJob.
 598     return (*group.jobs().begin())->GetLoadState();
 599   }
 600
 601   if (group.CanUseAdditionalSocketSlot(max_sockets_per_group_))
 602     return LOAD_STATE_WAITING_FOR_STALLED_SOCKET_POOL;
 603   return LOAD_STATE_WAITING_FOR_AVAILABLE_SOCKET;
 604 }
 605
 606 scoped_ptr<base::DictionaryValue> ClientSocketPoolBaseHelper::GetInfoAsValue(
 607     const std::string& name, const std::string& type) const {
 608   scoped_ptr<base::DictionaryValue> dict(new base::DictionaryValue());
 609   dict->SetString("name", name);
 610   dict->SetString("type", type);
 611   dict->SetInteger("handed_out_socket_count", handed_out_socket_count_);
 612   dict->SetInteger("connecting_socket_count", connecting_socket_count_);
 613   dict->SetInteger("idle_socket_count", idle_socket_count_);
 614   dict->SetInteger("max_socket_count", max_sockets_);
 615   dict->SetInteger("max_sockets_per_group", max_sockets_per_group_);
 616   dict->SetInteger("pool_generation_number", pool_generation_number_);
 617
 618   if (group_map_.empty())
 619     return dict.Pass();
 620
 621   base::DictionaryValue* all_groups_dict = new base::DictionaryValue();
 622   for (GroupMap::const_iterator it = group_map_.begin();
 623        it != group_map_.end(); it++) {
 624     const Group* group = it->second;
 625     base::DictionaryValue* group_dict = new base::DictionaryValue();
 626
 627     group_dict->SetInteger("pending_request_count",
 628                            group->pending_request_count());
 629     if (group->has_pending_requests()) {
 630       group_dict->SetString(
 631           "top_pending_priority",
 632           RequestPriorityToString(group->TopPendingPriority()));
 633     }
 634
 635     group_dict->SetInteger("active_socket_count", group->active_socket_count());
 636
 637     base::ListValue* idle_socket_list = new base::ListValue();
 638     std::list<IdleSocket>::const_iterator idle_socket;
 639     for (idle_socket = group->idle_sockets().begin();
 640          idle_socket != group->idle_sockets().end();
 641          idle_socket++) {
 642       int source_id = idle_socket->socket->NetLog().source().id;
 643       idle_socket_list->Append(new base::FundamentalValue(source_id));
 644     }
 645     group_dict->Set("idle_sockets", idle_socket_list);
 646
 647     base::ListValue* connect_jobs_list = new base::ListValue();
 648     std::list<ConnectJob*>::const_iterator job = group->jobs().begin();
 649     for (job = group->jobs().begin(); job != group->jobs().end(); job++) {
 650       int source_id = (*job)->net_log().source().id;
 651       connect_jobs_list->Append(new base::FundamentalValue(source_id));
 652     }
 653     group_dict->Set("connect_jobs", connect_jobs_list);
 654
 655     group_dict->SetBoolean("is_stalled", group->CanUseAdditionalSocketSlot(
 656                                              max_sockets_per_group_));
 657     group_dict->SetBoolean("backup_job_timer_is_running",
 658                            group->BackupJobTimerIsRunning());
 659
 660     all_groups_dict->SetWithoutPathExpansion(it->first, group_dict);
 661   }
 662   dict->Set("groups", all_groups_dict);
 663   return dict.Pass();
 664 }
 665
 666 bool ClientSocketPoolBaseHelper::IdleSocket::IsUsable() const {
 667   if (socket->WasEverUsed())
 668     return socket->IsConnectedAndIdle();
 669   return socket->IsConnected();
 670 }
 671
 672 bool ClientSocketPoolBaseHelper::IdleSocket::ShouldCleanup(
 673     base::TimeTicks now,
 674     base::TimeDelta timeout) const {
 675   bool timed_out = (now - start_time) >= timeout;
 676   if (timed_out)
 677     return true;
 678   return !IsUsable();
 679 }
 680
 681 void ClientSocketPoolBaseHelper::CleanupIdleSockets(bool force) {
 682   if (idle_socket_count_ == 0)
 683     return;
 684
 685   // Current time value. Retrieving it once at the function start rather than
 686   // inside the inner loop, since it shouldn't change by any meaningful amount.
 687   base::TimeTicks now = base::TimeTicks::Now();
 688
 689   GroupMap::iterator i = group_map_.begin();
 690   while (i != group_map_.end()) {
 691     Group* group = i->second;
 692
 693     std::list<IdleSocket>::iterator j = group->mutable_idle_sockets()->begin();
 694     while (j != group->idle_sockets().end()) {
 695       base::TimeDelta timeout =
 696           j->socket->WasEverUsed() ?
 697           used_idle_socket_timeout_ : unused_idle_socket_timeout_;
 698       if (force || j->ShouldCleanup(now, timeout)) {
 699         delete j->socket;
 700         j = group->mutable_idle_sockets()->erase(j);
 701         DecrementIdleCount();
 702       } else {
 703         ++j;
 704       }
 705     }
 706
 707     // Delete group if no longer needed.
 708     if (group->IsEmpty()) {
 709       RemoveGroup(i++);
 710     } else {
 711       ++i;
 712     }
 713   }
 714 }
 715
 716 ClientSocketPoolBaseHelper::Group* ClientSocketPoolBaseHelper::GetOrCreateGroup(
 717     const std::string& group_name) {
 718   GroupMap::iterator it = group_map_.find(group_name);
 719   if (it != group_map_.end())
 720     return it->second;
 721   Group* group = new Group;
 722   group_map_[group_name] = group;
 723   return group;
 724 }
 725
 726 void ClientSocketPoolBaseHelper::RemoveGroup(const std::string& group_name) {
 727   GroupMap::iterator it = group_map_.find(group_name);
 728   CHECK(it != group_map_.end());
 729
 730   RemoveGroup(it);
 731 }
 732
 733 void ClientSocketPoolBaseHelper::RemoveGroup(GroupMap::iterator it) {
 734   delete it->second;
 735   group_map_.erase(it);
 736 }
 737
 738 // static
 739 bool ClientSocketPoolBaseHelper::connect_backup_jobs_enabled() {
 740   return g_connect_backup_jobs_enabled;
 741 }
 742
 743 // static
 744 bool ClientSocketPoolBaseHelper::set_connect_backup_jobs_enabled(bool enabled) {
 745   bool old_value = g_connect_backup_jobs_enabled;
 746   g_connect_backup_jobs_enabled = enabled;
 747   return old_value;
 748 }
 749
 750 void ClientSocketPoolBaseHelper::EnableConnectBackupJobs() {
 751   connect_backup_jobs_enabled_ = g_connect_backup_jobs_enabled;
 752 }
 753
 754 void ClientSocketPoolBaseHelper::IncrementIdleCount() {
 755   if (++idle_socket_count_ == 1 && use_cleanup_timer_)
 756     StartIdleSocketTimer();
 757 }
 758
 759 void ClientSocketPoolBaseHelper::DecrementIdleCount() {
 760   if (--idle_socket_count_ == 0)
 761     timer_.Stop();
 762 }
 763
 764 // static
 765 bool ClientSocketPoolBaseHelper::cleanup_timer_enabled() {
 766   return g_cleanup_timer_enabled;
 767 }
 768
 769 // static
 770 bool ClientSocketPoolBaseHelper::set_cleanup_timer_enabled(bool enabled) {
 771   bool old_value = g_cleanup_timer_enabled;
 772   g_cleanup_timer_enabled = enabled;
 773   return old_value;
 774 }
 775
 776 void ClientSocketPoolBaseHelper::StartIdleSocketTimer() {
 777   timer_.Start(FROM_HERE, TimeDelta::FromSeconds(kCleanupInterval), this,
 778                &ClientSocketPoolBaseHelper::OnCleanupTimerFired);
 779 }
 780
 781 void ClientSocketPoolBaseHelper::ReleaseSocket(const std::string& group_name,
 782                                                scoped_ptr<StreamSocket> socket,
 783                                                int id) {
 784   GroupMap::iterator i = group_map_.find(group_name);
 785   CHECK(i != group_map_.end());
 786
 787   Group* group = i->second;
 788
 789   CHECK_GT(handed_out_socket_count_, 0);
 790   handed_out_socket_count_--;
 791
 792   CHECK_GT(group->active_socket_count(), 0);
 793   group->DecrementActiveSocketCount();
 794
 795   const bool can_reuse = socket->IsConnectedAndIdle() &&
 796       id == pool_generation_number_;
 797   if (can_reuse) {
 798     // Add it to the idle list.
 799     AddIdleSocket(socket.Pass(), group);
 800     OnAvailableSocketSlot(group_name, group);
 801   } else {
 802     socket.reset();
 803   }
 804
 805   CheckForStalledSocketGroups();
 806 }
 807
 808 void ClientSocketPoolBaseHelper::CheckForStalledSocketGroups() {
 809   // If we have idle sockets, see if we can give one to the top-stalled group.
 810   std::string top_group_name;
 811   Group* top_group = NULL;
 812   if (!FindTopStalledGroup(&top_group, &top_group_name)) {
 813     // There may still be a stalled group in a lower level pool.
 814     for (std::set<LowerLayeredPool*>::iterator it = lower_pools_.begin();
 815          it != lower_pools_.end();
 816          ++it) {
 817        if ((*it)->IsStalled()) {
 818          CloseOneIdleSocket();
 819          break;
 820        }
 821     }
 822     return;
 823   }
 824
 825   if (ReachedMaxSocketsLimit()) {
 826     if (idle_socket_count() > 0) {
 827       CloseOneIdleSocket();
 828     } else {
 829       // We can't activate more sockets since we're already at our global
 830       // limit.
 831       return;
 832     }
 833   }
 834
 835   // Note:  we don't loop on waking stalled groups.  If the stalled group is at
 836   //        its limit, may be left with other stalled groups that could be
 837   //        woken.  This isn't optimal, but there is no starvation, so to avoid
 838   //        the looping we leave it at this.
 839   OnAvailableSocketSlot(top_group_name, top_group);
 840 }
 841
 842 // Search for the highest priority pending request, amongst the groups that
 843 // are not at the |max_sockets_per_group_| limit. Note: for requests with
 844 // the same priority, the winner is based on group hash ordering (and not
 845 // insertion order).
 846 bool ClientSocketPoolBaseHelper::FindTopStalledGroup(
 847     Group** group,
 848     std::string* group_name) const {
 849   CHECK((group && group_name) || (!group && !group_name));
 850   Group* top_group = NULL;
 851   const std::string* top_group_name = NULL;
 852   bool has_stalled_group = false;
 853   for (GroupMap::const_iterator i = group_map_.begin();
 854        i != group_map_.end(); ++i) {
 855     Group* curr_group = i->second;
 856     if (!curr_group->has_pending_requests())
 857       continue;
 858     if (curr_group->CanUseAdditionalSocketSlot(max_sockets_per_group_)) {
 859       if (!group)
 860         return true;
 861       has_stalled_group = true;
 862       bool has_higher_priority = !top_group ||
 863           curr_group->TopPendingPriority() > top_group->TopPendingPriority();
 864       if (has_higher_priority) {
 865         top_group = curr_group;
 866         top_group_name = &i->first;
 867       }
 868     }
 869   }
 870
 871   if (top_group) {
 872     CHECK(group);
 873     *group = top_group;
 874     *group_name = *top_group_name;
 875   } else {
 876     CHECK(!has_stalled_group);
 877   }
 878   return has_stalled_group;
 879 }
 880
 881 void ClientSocketPoolBaseHelper::OnConnectJobComplete(
 882     int result, ConnectJob* job) {
 883   DCHECK_NE(ERR_IO_PENDING, result);
 884   const std::string group_name = job->group_name();
 885   GroupMap::iterator group_it = group_map_.find(group_name);
 886   CHECK(group_it != group_map_.end());
 887   Group* group = group_it->second;
 888
 889   scoped_ptr<StreamSocket> socket = job->PassSocket();
 890
 891   // Copies of these are needed because |job| may be deleted before they are
 892   // accessed.
 893   BoundNetLog job_log = job->net_log();
 894   LoadTimingInfo::ConnectTiming connect_timing = job->connect_timing();
 895
 896   // RemoveConnectJob(job, _) must be called by all branches below;
 897   // otherwise, |job| will be leaked.
 898
 899   if (result == OK) {
 900     DCHECK(socket.get());
 901     RemoveConnectJob(job, group);
 902     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 903     if (request) {
 904       LogBoundConnectJobToRequest(job_log.source(), *request);
 905       HandOutSocket(
 906           socket.Pass(), ClientSocketHandle::UNUSED, connect_timing,
 907           request->handle(), base::TimeDelta(), group, request->net_log());
 908       request->net_log().EndEvent(NetLog::TYPE_SOCKET_POOL);
 909       InvokeUserCallbackLater(request->handle(), request->callback(), result);
 910     } else {
 911       AddIdleSocket(socket.Pass(), group);
 912       OnAvailableSocketSlot(group_name, group);
 913       CheckForStalledSocketGroups();
 914     }
 915   } else {
 916     // If we got a socket, it must contain error information so pass that
 917     // up so that the caller can retrieve it.
 918     bool handed_out_socket = false;
 919     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 920     if (request) {
 921       LogBoundConnectJobToRequest(job_log.source(), *request);
 922       job->GetAdditionalErrorState(request->handle());
 923       RemoveConnectJob(job, group);
 924       if (socket.get()) {
 925         handed_out_socket = true;
 926         HandOutSocket(socket.Pass(), ClientSocketHandle::UNUSED,
 927                       connect_timing, request->handle(), base::TimeDelta(),
 928                       group, request->net_log());
 929       }
 930       request->net_log().EndEventWithNetErrorCode(
 931           NetLog::TYPE_SOCKET_POOL, result);
 932       InvokeUserCallbackLater(request->handle(), request->callback(), result);
 933     } else {
 934       RemoveConnectJob(job, group);
 935     }
 936     if (!handed_out_socket) {
 937       OnAvailableSocketSlot(group_name, group);
 938       CheckForStalledSocketGroups();
 939     }
 940   }
 941 }
 942
 943 void ClientSocketPoolBaseHelper::OnIPAddressChanged() {
 944   FlushWithError(ERR_NETWORK_CHANGED);
 945 }
 946
 947 void ClientSocketPoolBaseHelper::FlushWithError(int error) {
 948   pool_generation_number_++;
 949   CancelAllConnectJobs();
 950   CloseIdleSockets();
 951   CancelAllRequestsWithError(error);
 952 }
 953
 954 void ClientSocketPoolBaseHelper::RemoveConnectJob(ConnectJob* job,
 955                                                   Group* group) {
 956   CHECK_GT(connecting_socket_count_, 0);
 957   connecting_socket_count_--;
 958
 959   DCHECK(group);
 960   group->RemoveJob(job);
 961 }
 962
 963 void ClientSocketPoolBaseHelper::OnAvailableSocketSlot(
 964     const std::string& group_name, Group* group) {
 965   DCHECK(ContainsKey(group_map_, group_name));
 966   if (group->IsEmpty()) {
 967     RemoveGroup(group_name);
 968   } else if (group->has_pending_requests()) {
 969     ProcessPendingRequest(group_name, group);
 970   }
 971 }
 972
 973 void ClientSocketPoolBaseHelper::ProcessPendingRequest(
 974     const std::string& group_name, Group* group) {
 975   const Request* next_request = group->GetNextPendingRequest();
 976   DCHECK(next_request);
 977
 978   // If the group has no idle sockets, and can't make use of an additional slot,
 979   // either because it's at the limit or because it's at the socket per group
 980   // limit, then there's nothing to do.
 981   if (group->idle_sockets().empty() &&
 982       !group->CanUseAdditionalSocketSlot(max_sockets_per_group_)) {
 983     return;
 984   }
 985
 986   int rv = RequestSocketInternal(group_name, *next_request);
 987   if (rv != ERR_IO_PENDING) {
 988     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 989     DCHECK(request);
 990     if (group->IsEmpty())
 991       RemoveGroup(group_name);
 992
 993     request->net_log().EndEventWithNetErrorCode(NetLog::TYPE_SOCKET_POOL, rv);
 994     InvokeUserCallbackLater(request->handle(), request->callback(), rv);
 995   }
 996 }
 997
 998 void ClientSocketPoolBaseHelper::HandOutSocket(
 999     scoped_ptr<StreamSocket> socket,
1000     ClientSocketHandle::SocketReuseType reuse_type,
1001     const LoadTimingInfo::ConnectTiming& connect_timing,
1002     ClientSocketHandle* handle,
1003     base::TimeDelta idle_time,
1004     Group* group,
1005     const BoundNetLog& net_log) {
1006   DCHECK(socket);
1007   handle->SetSocket(socket.Pass());
1008   handle->set_reuse_type(reuse_type);
1009   handle->set_idle_time(idle_time);
1010   handle->set_pool_id(pool_generation_number_);
1011   handle->set_connect_timing(connect_timing);
1012
1013   if (handle->is_reused()) {
1014     net_log.AddEvent(
1015         NetLog::TYPE_SOCKET_POOL_REUSED_AN_EXISTING_SOCKET,
1016         NetLog::IntegerCallback(
1017             "idle_ms", static_cast<int>(idle_time.InMilliseconds())));
1018   }
1019
1020   net_log.AddEvent(
1021       NetLog::TYPE_SOCKET_POOL_BOUND_TO_SOCKET,
1022       handle->socket()->NetLog().source().ToEventParametersCallback());
1023
1024   handed_out_socket_count_++;
1025   group->IncrementActiveSocketCount();
1026 }
1027
1028 void ClientSocketPoolBaseHelper::AddIdleSocket(
1029     scoped_ptr<StreamSocket> socket,
1030     Group* group) {
1031   DCHECK(socket);
1032   IdleSocket idle_socket;
1033   idle_socket.socket = socket.release();
1034   idle_socket.start_time = base::TimeTicks::Now();
1035
1036   group->mutable_idle_sockets()->push_back(idle_socket);
1037   IncrementIdleCount();
1038 }
1039
1040 void ClientSocketPoolBaseHelper::CancelAllConnectJobs() {
1041   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end();) {
1042     Group* group = i->second;
1043     connecting_socket_count_ -= group->jobs().size();
1044     group->RemoveAllJobs();
1045
1046     // Delete group if no longer needed.
1047     if (group->IsEmpty()) {
1048       // RemoveGroup() will call .erase() which will invalidate the iterator,
1049       // but i will already have been incremented to a valid iterator before
1050       // RemoveGroup() is called.
1051       RemoveGroup(i++);
1052     } else {
1053       ++i;
1054     }
1055   }
1056   DCHECK_EQ(0, connecting_socket_count_);
1057 }
1058
1059 void ClientSocketPoolBaseHelper::CancelAllRequestsWithError(int error) {
1060   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end();) {
1061     Group* group = i->second;
1062
1063     while (true) {
1064       scoped_ptr<const Request> request = group->PopNextPendingRequest();
1065       if (!request)
1066         break;
1067       InvokeUserCallbackLater(request->handle(), request->callback(), error);
1068     }
1069
1070     // Delete group if no longer needed.
1071     if (group->IsEmpty()) {
1072       // RemoveGroup() will call .erase() which will invalidate the iterator,
1073       // but i will already have been incremented to a valid iterator before
1074       // RemoveGroup() is called.
1075       RemoveGroup(i++);
1076     } else {
1077       ++i;
1078     }
1079   }
1080 }
1081
1082 bool ClientSocketPoolBaseHelper::ReachedMaxSocketsLimit() const {
1083   // Each connecting socket will eventually connect and be handed out.
1084   int total = handed_out_socket_count_ + connecting_socket_count_ +
1085       idle_socket_count();
1086   // There can be more sockets than the limit since some requests can ignore
1087   // the limit
1088   if (total < max_sockets_)
1089     return false;
1090   return true;
1091 }
1092
1093 bool ClientSocketPoolBaseHelper::CloseOneIdleSocket() {
1094   if (idle_socket_count() == 0)
1095     return false;
1096   return CloseOneIdleSocketExceptInGroup(NULL);
1097 }
1098
1099 bool ClientSocketPoolBaseHelper::CloseOneIdleSocketExceptInGroup(
1100     const Group* exception_group) {
1101   CHECK_GT(idle_socket_count(), 0);
1102
1103   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end(); ++i) {
1104     Group* group = i->second;
1105     if (exception_group == group)
1106       continue;
1107     std::list<IdleSocket>* idle_sockets = group->mutable_idle_sockets();
1108
1109     if (!idle_sockets->empty()) {
1110       delete idle_sockets->front().socket;
1111       idle_sockets->pop_front();
1112       DecrementIdleCount();
1113       if (group->IsEmpty())
1114         RemoveGroup(i);
1115
1116       return true;
1117     }
1118   }
1119
1120   return false;
1121 }
1122
1123 bool ClientSocketPoolBaseHelper::CloseOneIdleConnectionInHigherLayeredPool() {
1124   // This pool doesn't have any idle sockets. It's possible that a pool at a
1125   // higher layer is holding one of this sockets active, but it's actually idle.
1126   // Query the higher layers.
1127   for (std::set<HigherLayeredPool*>::const_iterator it = higher_pools_.begin();
1128        it != higher_pools_.end(); ++it) {
1129     if ((*it)->CloseOneIdleConnection())
1130       return true;
1131   }
1132   return false;
1133 }
1134
1135 void ClientSocketPoolBaseHelper::InvokeUserCallbackLater(
1136     ClientSocketHandle* handle, const CompletionCallback& callback, int rv) {
1137   CHECK(!ContainsKey(pending_callback_map_, handle));
1138   pending_callback_map_[handle] = CallbackResultPair(callback, rv);
1139   base::ThreadTaskRunnerHandle::Get()->PostTask(
1140       FROM_HERE, base::Bind(&ClientSocketPoolBaseHelper::InvokeUserCallback,
1141                             weak_factory_.GetWeakPtr(), handle));
1142 }
1143
1144 void ClientSocketPoolBaseHelper::InvokeUserCallback(
1145     ClientSocketHandle* handle) {
1146   PendingCallbackMap::iterator it = pending_callback_map_.find(handle);
1147
1148   // Exit if the request has already been cancelled.
1149   if (it == pending_callback_map_.end())
1150     return;
1151
1152   CHECK(!handle->is_initialized());
1153   CompletionCallback callback = it->second.callback;
1154   int result = it->second.result;
1155   pending_callback_map_.erase(it);
1156   callback.Run(result);
1157 }
1158
1159 void ClientSocketPoolBaseHelper::TryToCloseSocketsInLayeredPools() {
1160   while (IsStalled()) {
1161     // Closing a socket will result in calling back into |this| to use the freed
1162     // socket slot, so nothing else is needed.
1163     if (!CloseOneIdleConnectionInHigherLayeredPool())
1164       return;
1165   }
1166 }
1167
1168 ClientSocketPoolBaseHelper::Group::Group()
1169     : unassigned_job_count_(0),
1170       pending_requests_(NUM_PRIORITIES),
1171       active_socket_count_(0) {}
1172
1173 ClientSocketPoolBaseHelper::Group::~Group() {
1174   DCHECK_EQ(0u, unassigned_job_count_);
1175 }
1176
1177 void ClientSocketPoolBaseHelper::Group::StartBackupJobTimer(
1178     const std::string& group_name,
1179     ClientSocketPoolBaseHelper* pool) {
1180   // Only allow one timer to run at a time.
1181   if (BackupJobTimerIsRunning())
1182     return;
1183
1184   // Unretained here is okay because |backup_job_timer_| is
1185   // automatically cancelled when it's destroyed.
1186   backup_job_timer_.Start(
1187       FROM_HERE, pool->ConnectRetryInterval(),
1188       base::Bind(&Group::OnBackupJobTimerFired, base::Unretained(this),
1189                  group_name, pool));
1190 }
1191
1192 bool ClientSocketPoolBaseHelper::Group::BackupJobTimerIsRunning() const {
1193   return backup_job_timer_.IsRunning();
1194 }
1195
1196 bool ClientSocketPoolBaseHelper::Group::TryToUseUnassignedConnectJob() {
1197   SanityCheck();
1198
1199   if (unassigned_job_count_ == 0)
1200     return false;
1201   --unassigned_job_count_;
1202   return true;
1203 }
1204
1205 void ClientSocketPoolBaseHelper::Group::AddJob(scoped_ptr<ConnectJob> job,
1206                                                bool is_preconnect) {
1207   SanityCheck();
1208
1209   if (is_preconnect)
1210     ++unassigned_job_count_;
1211   jobs_.push_back(job.release());
1212 }
1213
1214 void ClientSocketPoolBaseHelper::Group::RemoveJob(ConnectJob* job) {
1215   scoped_ptr<ConnectJob> owned_job(job);
1216   SanityCheck();
1217
1218   // Check that |job| is in the list.
1219   DCHECK_EQ(*std::find(jobs_.begin(), jobs_.end(), job), job);
1220   jobs_.remove(job);
1221   size_t job_count = jobs_.size();
1222   if (job_count < unassigned_job_count_)
1223     unassigned_job_count_ = job_count;
1224
1225   // If we've got no more jobs for this group, then we no longer need a
1226   // backup job either.
1227   if (jobs_.empty())
1228     backup_job_timer_.Stop();
1229 }
1230
1231 void ClientSocketPoolBaseHelper::Group::OnBackupJobTimerFired(
1232     std::string group_name,
1233     ClientSocketPoolBaseHelper* pool) {
1234   // If there are no more jobs pending, there is no work to do.
1235   // If we've done our cleanups correctly, this should not happen.
1236   if (jobs_.empty()) {
1237     NOTREACHED();
1238     return;
1239   }
1240
1241   // If our old job is waiting on DNS, or if we can't create any sockets
1242   // right now due to limits, just reset the timer.
1243   if (pool->ReachedMaxSocketsLimit() ||
1244       !HasAvailableSocketSlot(pool->max_sockets_per_group_) ||
1245       (*jobs_.begin())->GetLoadState() == LOAD_STATE_RESOLVING_HOST) {
1246     StartBackupJobTimer(group_name, pool);
1247     return;
1248   }
1249
1250   if (pending_requests_.empty())
1251     return;
1252
1253   scoped_ptr<ConnectJob> backup_job =
1254       pool->connect_job_factory_->NewConnectJob(
1255           group_name, *pending_requests_.FirstMax().value(), pool);
1256   backup_job->net_log().AddEvent(NetLog::TYPE_BACKUP_CONNECT_JOB_CREATED);
1257   int rv = backup_job->Connect();
1258   pool->connecting_socket_count_++;
1259   ConnectJob* raw_backup_job = backup_job.get();
1260   AddJob(backup_job.Pass(), false);
1261   if (rv != ERR_IO_PENDING)
1262     pool->OnConnectJobComplete(rv, raw_backup_job);
1263 }
1264
1265 void ClientSocketPoolBaseHelper::Group::SanityCheck() {
1266   DCHECK_LE(unassigned_job_count_, jobs_.size());
1267 }
1268
1269 void ClientSocketPoolBaseHelper::Group::RemoveAllJobs() {
1270   SanityCheck();
1271
1272   // Delete active jobs.
1273   STLDeleteElements(&jobs_);
1274   unassigned_job_count_ = 0;
1275
1276   // Stop backup job timer.
1277   backup_job_timer_.Stop();
1278 }
1279
1280 const ClientSocketPoolBaseHelper::Request*
1281 ClientSocketPoolBaseHelper::Group::GetNextPendingRequest() const {
1282   return
1283       pending_requests_.empty() ? NULL : pending_requests_.FirstMax().value();
1284 }
1285
1286 bool ClientSocketPoolBaseHelper::Group::HasConnectJobForHandle(
1287     const ClientSocketHandle* handle) const {
1288   // Search the first |jobs_.size()| pending requests for |handle|.
1289   // If it's farther back in the deque than that, it doesn't have a
1290   // corresponding ConnectJob.
1291   size_t i = 0;
1292   for (RequestQueue::Pointer pointer = pending_requests_.FirstMax();
1293        !pointer.is_null() && i < jobs_.size();
1294        pointer = pending_requests_.GetNextTowardsLastMin(pointer), ++i) {
1295     if (pointer.value()->handle() == handle)
1296       return true;
1297   }
1298   return false;
1299 }
1300
1301 void ClientSocketPoolBaseHelper::Group::InsertPendingRequest(
1302     scoped_ptr<const Request> request) {
1303   // This value must be cached before we release |request|.
1304   RequestPriority priority = request->priority();
1305   if (request->ignore_limits()) {
1306     // Put requests with ignore_limits == true (which should have
1307     // priority == MAXIMUM_PRIORITY) ahead of other requests with
1308     // MAXIMUM_PRIORITY.
1309     DCHECK_EQ(priority, MAXIMUM_PRIORITY);
1310     pending_requests_.InsertAtFront(request.release(), priority);
1311   } else {
1312     pending_requests_.Insert(request.release(), priority);
1313   }
1314 }
1315
1316 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1317 ClientSocketPoolBaseHelper::Group::PopNextPendingRequest() {
1318   if (pending_requests_.empty())
1319     return scoped_ptr<const ClientSocketPoolBaseHelper::Request>();
1320   return RemovePendingRequest(pending_requests_.FirstMax());
1321 }
1322
1323 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1324 ClientSocketPoolBaseHelper::Group::FindAndRemovePendingRequest(
1325     ClientSocketHandle* handle) {
1326   for (RequestQueue::Pointer pointer = pending_requests_.FirstMax();
1327        !pointer.is_null();
1328        pointer = pending_requests_.GetNextTowardsLastMin(pointer)) {
1329     if (pointer.value()->handle() == handle) {
1330       scoped_ptr<const Request> request = RemovePendingRequest(pointer);
1331       return request.Pass();
1332     }
1333   }
1334   return scoped_ptr<const ClientSocketPoolBaseHelper::Request>();
1335 }
1336
1337 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1338 ClientSocketPoolBaseHelper::Group::RemovePendingRequest(
1339     const RequestQueue::Pointer& pointer) {
1340   // TODO(eroman): Temporary for debugging http://crbug.com/467797.
1341   CHECK(!pointer.is_null());
1342   scoped_ptr<const Request> request(pointer.value());
1343   pending_requests_.Erase(pointer);
1344   // If there are no more requests, kill the backup timer.
1345   if (pending_requests_.empty())
1346     backup_job_timer_.Stop();
1347   request->CrashIfInvalid();
1348   return request.Pass();
1349 }
1350
1351 }  // namespace internal
1352
1353 }  // namespace net