net/socket/client_socket_pool_base.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/socket/client_socket_pool_base.h"
   6
   7 #include "base/compiler_specific.h"
   8 #include "base/format_macros.h"
   9 #include "base/logging.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/metrics/stats_counters.h"
  12 #include "base/profiler/scoped_tracker.h"
  13 #include "base/stl_util.h"
  14 #include "base/strings/string_util.h"
  15 #include "base/time/time.h"
  16 #include "base/values.h"
  17 #include "net/base/net_errors.h"
  18 #include "net/base/net_log.h"
  19
  20 using base::TimeDelta;
  21
  22 namespace net {
  23
  24 namespace {
  25
  26 // Indicate whether we should enable idle socket cleanup timer. When timer is
  27 // disabled, sockets are closed next time a socket request is made.
  28 bool g_cleanup_timer_enabled = true;
  29
  30 // The timeout value, in seconds, used to clean up idle sockets that can't be
  31 // reused.
  32 //
  33 // Note: It's important to close idle sockets that have received data as soon
  34 // as possible because the received data may cause BSOD on Windows XP under
  35 // some conditions.  See http://crbug.com/4606.
  36 const int kCleanupInterval = 10;  // DO NOT INCREASE THIS TIMEOUT.
  37
  38 // Indicate whether or not we should establish a new transport layer connection
  39 // after a certain timeout has passed without receiving an ACK.
  40 bool g_connect_backup_jobs_enabled = true;
  41
  42 }  // namespace
  43
  44 ConnectJob::ConnectJob(const std::string& group_name,
  45                        base::TimeDelta timeout_duration,
  46                        RequestPriority priority,
  47                        Delegate* delegate,
  48                        const BoundNetLog& net_log)
  49     : group_name_(group_name),
  50       timeout_duration_(timeout_duration),
  51       priority_(priority),
  52       delegate_(delegate),
  53       net_log_(net_log),
  54       idle_(true) {
  55   DCHECK(!group_name.empty());
  56   DCHECK(delegate);
  57   net_log.BeginEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB,
  58                      NetLog::StringCallback("group_name", &group_name_));
  59 }
  60
  61 ConnectJob::~ConnectJob() {
  62   net_log().EndEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB);
  63 }
  64
  65 scoped_ptr<StreamSocket> ConnectJob::PassSocket() {
  66   return socket_.Pass();
  67 }
  68
  69 int ConnectJob::Connect() {
  70   if (timeout_duration_ != base::TimeDelta())
  71     timer_.Start(FROM_HERE, timeout_duration_, this, &ConnectJob::OnTimeout);
  72
  73   idle_ = false;
  74
  75   LogConnectStart();
  76
  77   int rv = ConnectInternal();
  78
  79   if (rv != ERR_IO_PENDING) {
  80     LogConnectCompletion(rv);
  81     delegate_ = NULL;
  82   }
  83
  84   return rv;
  85 }
  86
  87 void ConnectJob::SetSocket(scoped_ptr<StreamSocket> socket) {
  88   if (socket) {
  89     net_log().AddEvent(NetLog::TYPE_CONNECT_JOB_SET_SOCKET,
  90                        socket->NetLog().source().ToEventParametersCallback());
  91   }
  92   socket_ = socket.Pass();
  93 }
  94
  95 void ConnectJob::NotifyDelegateOfCompletion(int rv) {
  96   // The delegate will own |this|.
  97   Delegate* delegate = delegate_;
  98   delegate_ = NULL;
  99
 100   LogConnectCompletion(rv);
 101   delegate->OnConnectJobComplete(rv, this);
 102 }
 103
 104 void ConnectJob::ResetTimer(base::TimeDelta remaining_time) {
 105   timer_.Stop();
 106   timer_.Start(FROM_HERE, remaining_time, this, &ConnectJob::OnTimeout);
 107 }
 108
 109 void ConnectJob::LogConnectStart() {
 110   connect_timing_.connect_start = base::TimeTicks::Now();
 111   net_log().BeginEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_CONNECT);
 112 }
 113
 114 void ConnectJob::LogConnectCompletion(int net_error) {
 115   connect_timing_.connect_end = base::TimeTicks::Now();
 116   net_log().EndEventWithNetErrorCode(
 117       NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_CONNECT, net_error);
 118 }
 119
 120 void ConnectJob::OnTimeout() {
 121   // Make sure the socket is NULL before calling into |delegate|.
 122   SetSocket(scoped_ptr<StreamSocket>());
 123
 124   net_log_.AddEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_TIMED_OUT);
 125
 126   NotifyDelegateOfCompletion(ERR_TIMED_OUT);
 127 }
 128
 129 namespace internal {
 130
 131 ClientSocketPoolBaseHelper::Request::Request(
 132     ClientSocketHandle* handle,
 133     const CompletionCallback& callback,
 134     RequestPriority priority,
 135     bool ignore_limits,
 136     Flags flags,
 137     const BoundNetLog& net_log)
 138     : handle_(handle),
 139       callback_(callback),
 140       priority_(priority),
 141       ignore_limits_(ignore_limits),
 142       flags_(flags),
 143       net_log_(net_log) {
 144   if (ignore_limits_)
 145     DCHECK_EQ(priority_, MAXIMUM_PRIORITY);
 146 }
 147
 148 ClientSocketPoolBaseHelper::Request::~Request() {}
 149
 150 ClientSocketPoolBaseHelper::ClientSocketPoolBaseHelper(
 151     HigherLayeredPool* pool,
 152     int max_sockets,
 153     int max_sockets_per_group,
 154     base::TimeDelta unused_idle_socket_timeout,
 155     base::TimeDelta used_idle_socket_timeout,
 156     ConnectJobFactory* connect_job_factory)
 157     : idle_socket_count_(0),
 158       connecting_socket_count_(0),
 159       handed_out_socket_count_(0),
 160       max_sockets_(max_sockets),
 161       max_sockets_per_group_(max_sockets_per_group),
 162       use_cleanup_timer_(g_cleanup_timer_enabled),
 163       unused_idle_socket_timeout_(unused_idle_socket_timeout),
 164       used_idle_socket_timeout_(used_idle_socket_timeout),
 165       connect_job_factory_(connect_job_factory),
 166       connect_backup_jobs_enabled_(false),
 167       pool_generation_number_(0),
 168       pool_(pool),
 169       weak_factory_(this) {
 170   DCHECK_LE(0, max_sockets_per_group);
 171   DCHECK_LE(max_sockets_per_group, max_sockets);
 172
 173   NetworkChangeNotifier::AddIPAddressObserver(this);
 174 }
 175
 176 ClientSocketPoolBaseHelper::~ClientSocketPoolBaseHelper() {
 177   // Clean up any idle sockets and pending connect jobs.  Assert that we have no
 178   // remaining active sockets or pending requests.  They should have all been
 179   // cleaned up prior to |this| being destroyed.
 180   FlushWithError(ERR_ABORTED);
 181   DCHECK(group_map_.empty());
 182   DCHECK(pending_callback_map_.empty());
 183   DCHECK_EQ(0, connecting_socket_count_);
 184   CHECK(higher_pools_.empty());
 185
 186   NetworkChangeNotifier::RemoveIPAddressObserver(this);
 187
 188   // Remove from lower layer pools.
 189   for (std::set<LowerLayeredPool*>::iterator it = lower_pools_.begin();
 190        it != lower_pools_.end();
 191        ++it) {
 192     (*it)->RemoveHigherLayeredPool(pool_);
 193   }
 194 }
 195
 196 ClientSocketPoolBaseHelper::CallbackResultPair::CallbackResultPair()
 197     : result(OK) {
 198 }
 199
 200 ClientSocketPoolBaseHelper::CallbackResultPair::CallbackResultPair(
 201     const CompletionCallback& callback_in, int result_in)
 202     : callback(callback_in),
 203       result(result_in) {
 204 }
 205
 206 ClientSocketPoolBaseHelper::CallbackResultPair::~CallbackResultPair() {}
 207
 208 bool ClientSocketPoolBaseHelper::IsStalled() const {
 209   // If a lower layer pool is stalled, consider |this| stalled as well.
 210   for (std::set<LowerLayeredPool*>::const_iterator it = lower_pools_.begin();
 211        it != lower_pools_.end();
 212        ++it) {
 213     if ((*it)->IsStalled())
 214       return true;
 215   }
 216
 217   // If fewer than |max_sockets_| are in use, then clearly |this| is not
 218   // stalled.
 219   if ((handed_out_socket_count_ + connecting_socket_count_) < max_sockets_)
 220     return false;
 221   // So in order to be stalled, |this| must be using at least |max_sockets_| AND
 222   // |this| must have a request that is actually stalled on the global socket
 223   // limit.  To find such a request, look for a group that has more requests
 224   // than jobs AND where the number of sockets is less than
 225   // |max_sockets_per_group_|.  (If the number of sockets is equal to
 226   // |max_sockets_per_group_|, then the request is stalled on the group limit,
 227   // which does not count.)
 228   for (GroupMap::const_iterator it = group_map_.begin();
 229        it != group_map_.end(); ++it) {
 230     if (it->second->IsStalledOnPoolMaxSockets(max_sockets_per_group_))
 231       return true;
 232   }
 233   return false;
 234 }
 235
 236 void ClientSocketPoolBaseHelper::AddLowerLayeredPool(
 237     LowerLayeredPool* lower_pool) {
 238   DCHECK(pool_);
 239   CHECK(!ContainsKey(lower_pools_, lower_pool));
 240   lower_pools_.insert(lower_pool);
 241   lower_pool->AddHigherLayeredPool(pool_);
 242 }
 243
 244 void ClientSocketPoolBaseHelper::AddHigherLayeredPool(
 245     HigherLayeredPool* higher_pool) {
 246   CHECK(higher_pool);
 247   CHECK(!ContainsKey(higher_pools_, higher_pool));
 248   higher_pools_.insert(higher_pool);
 249 }
 250
 251 void ClientSocketPoolBaseHelper::RemoveHigherLayeredPool(
 252     HigherLayeredPool* higher_pool) {
 253   CHECK(higher_pool);
 254   CHECK(ContainsKey(higher_pools_, higher_pool));
 255   higher_pools_.erase(higher_pool);
 256 }
 257
 258 int ClientSocketPoolBaseHelper::RequestSocket(
 259     const std::string& group_name,
 260     scoped_ptr<const Request> request) {
 261   CHECK(!request->callback().is_null());
 262   CHECK(request->handle());
 263
 264   // Cleanup any timed-out idle sockets if no timer is used.
 265   if (!use_cleanup_timer_)
 266     CleanupIdleSockets(false);
 267
 268   request->net_log().BeginEvent(NetLog::TYPE_SOCKET_POOL);
 269   Group* group = GetOrCreateGroup(group_name);
 270
 271   int rv = RequestSocketInternal(group_name, *request);
 272   if (rv != ERR_IO_PENDING) {
 273     request->net_log().EndEventWithNetErrorCode(NetLog::TYPE_SOCKET_POOL, rv);
 274     CHECK(!request->handle()->is_initialized());
 275     request.reset();
 276   } else {
 277     group->InsertPendingRequest(request.Pass());
 278     // Have to do this asynchronously, as closing sockets in higher level pools
 279     // call back in to |this|, which will cause all sorts of fun and exciting
 280     // re-entrancy issues if the socket pool is doing something else at the
 281     // time.
 282     if (group->IsStalledOnPoolMaxSockets(max_sockets_per_group_)) {
 283       base::MessageLoop::current()->PostTask(
 284           FROM_HERE,
 285           base::Bind(
 286               &ClientSocketPoolBaseHelper::TryToCloseSocketsInLayeredPools,
 287               weak_factory_.GetWeakPtr()));
 288     }
 289   }
 290   return rv;
 291 }
 292
 293 void ClientSocketPoolBaseHelper::RequestSockets(
 294     const std::string& group_name,
 295     const Request& request,
 296     int num_sockets) {
 297   DCHECK(request.callback().is_null());
 298   DCHECK(!request.handle());
 299
 300   // Cleanup any timed out idle sockets if no timer is used.
 301   if (!use_cleanup_timer_)
 302     CleanupIdleSockets(false);
 303
 304   if (num_sockets > max_sockets_per_group_) {
 305     num_sockets = max_sockets_per_group_;
 306   }
 307
 308   request.net_log().BeginEvent(
 309       NetLog::TYPE_SOCKET_POOL_CONNECTING_N_SOCKETS,
 310       NetLog::IntegerCallback("num_sockets", num_sockets));
 311
 312   Group* group = GetOrCreateGroup(group_name);
 313
 314   // RequestSocketsInternal() may delete the group.
 315   bool deleted_group = false;
 316
 317   int rv = OK;
 318   for (int num_iterations_left = num_sockets;
 319        group->NumActiveSocketSlots() < num_sockets &&
 320        num_iterations_left > 0 ; num_iterations_left--) {
 321     rv = RequestSocketInternal(group_name, request);
 322     if (rv < 0 && rv != ERR_IO_PENDING) {
 323       // We're encountering a synchronous error.  Give up.
 324       if (!ContainsKey(group_map_, group_name))
 325         deleted_group = true;
 326       break;
 327     }
 328     if (!ContainsKey(group_map_, group_name)) {
 329       // Unexpected.  The group should only be getting deleted on synchronous
 330       // error.
 331       NOTREACHED();
 332       deleted_group = true;
 333       break;
 334     }
 335   }
 336
 337   if (!deleted_group && group->IsEmpty())
 338     RemoveGroup(group_name);
 339
 340   if (rv == ERR_IO_PENDING)
 341     rv = OK;
 342   request.net_log().EndEventWithNetErrorCode(
 343       NetLog::TYPE_SOCKET_POOL_CONNECTING_N_SOCKETS, rv);
 344 }
 345
 346 int ClientSocketPoolBaseHelper::RequestSocketInternal(
 347     const std::string& group_name,
 348     const Request& request) {
 349   ClientSocketHandle* const handle = request.handle();
 350   const bool preconnecting = !handle;
 351   Group* group = GetOrCreateGroup(group_name);
 352
 353   if (!(request.flags() & NO_IDLE_SOCKETS)) {
 354     // Try to reuse a socket.
 355     if (AssignIdleSocketToRequest(request, group))
 356       return OK;
 357   }
 358
 359   // If there are more ConnectJobs than pending requests, don't need to do
 360   // anything.  Can just wait for the extra job to connect, and then assign it
 361   // to the request.
 362   if (!preconnecting && group->TryToUseUnassignedConnectJob())
 363     return ERR_IO_PENDING;
 364
 365   // Can we make another active socket now?
 366   if (!group->HasAvailableSocketSlot(max_sockets_per_group_) &&
 367       !request.ignore_limits()) {
 368     // TODO(willchan): Consider whether or not we need to close a socket in a
 369     // higher layered group. I don't think this makes sense since we would just
 370     // reuse that socket then if we needed one and wouldn't make it down to this
 371     // layer.
 372     request.net_log().AddEvent(
 373         NetLog::TYPE_SOCKET_POOL_STALLED_MAX_SOCKETS_PER_GROUP);
 374     return ERR_IO_PENDING;
 375   }
 376
 377   if (ReachedMaxSocketsLimit() && !request.ignore_limits()) {
 378     // NOTE(mmenke):  Wonder if we really need different code for each case
 379     // here.  Only reason for them now seems to be preconnects.
 380     if (idle_socket_count() > 0) {
 381       // There's an idle socket in this pool. Either that's because there's
 382       // still one in this group, but we got here due to preconnecting bypassing
 383       // idle sockets, or because there's an idle socket in another group.
 384       bool closed = CloseOneIdleSocketExceptInGroup(group);
 385       if (preconnecting && !closed)
 386         return ERR_PRECONNECT_MAX_SOCKET_LIMIT;
 387     } else {
 388       // We could check if we really have a stalled group here, but it requires
 389       // a scan of all groups, so just flip a flag here, and do the check later.
 390       request.net_log().AddEvent(NetLog::TYPE_SOCKET_POOL_STALLED_MAX_SOCKETS);
 391       return ERR_IO_PENDING;
 392     }
 393   }
 394
 395   // We couldn't find a socket to reuse, and there's space to allocate one,
 396   // so allocate and connect a new one.
 397   scoped_ptr<ConnectJob> connect_job(
 398       connect_job_factory_->NewConnectJob(group_name, request, this));
 399
 400   int rv = connect_job->Connect();
 401   if (rv == OK) {
 402     LogBoundConnectJobToRequest(connect_job->net_log().source(), request);
 403     if (!preconnecting) {
 404       HandOutSocket(connect_job->PassSocket(), ClientSocketHandle::UNUSED,
 405                     connect_job->connect_timing(), handle, base::TimeDelta(),
 406                     group, request.net_log());
 407     } else {
 408       AddIdleSocket(connect_job->PassSocket(), group);
 409     }
 410   } else if (rv == ERR_IO_PENDING) {
 411     // If we don't have any sockets in this group, set a timer for potentially
 412     // creating a new one.  If the SYN is lost, this backup socket may complete
 413     // before the slow socket, improving end user latency.
 414     if (connect_backup_jobs_enabled_ && group->IsEmpty()) {
 415       group->StartBackupJobTimer(group_name, this);
 416     }
 417
 418     connecting_socket_count_++;
 419
 420     group->AddJob(connect_job.Pass(), preconnecting);
 421   } else {
 422     LogBoundConnectJobToRequest(connect_job->net_log().source(), request);
 423     scoped_ptr<StreamSocket> error_socket;
 424     if (!preconnecting) {
 425       DCHECK(handle);
 426       connect_job->GetAdditionalErrorState(handle);
 427       error_socket = connect_job->PassSocket();
 428     }
 429     if (error_socket) {
 430       HandOutSocket(error_socket.Pass(), ClientSocketHandle::UNUSED,
 431                     connect_job->connect_timing(), handle, base::TimeDelta(),
 432                     group, request.net_log());
 433     } else if (group->IsEmpty()) {
 434       RemoveGroup(group_name);
 435     }
 436   }
 437
 438   return rv;
 439 }
 440
 441 bool ClientSocketPoolBaseHelper::AssignIdleSocketToRequest(
 442     const Request& request, Group* group) {
 443   std::list<IdleSocket>* idle_sockets = group->mutable_idle_sockets();
 444   std::list<IdleSocket>::iterator idle_socket_it = idle_sockets->end();
 445
 446   // Iterate through the idle sockets forwards (oldest to newest)
 447   //   * Delete any disconnected ones.
 448   //   * If we find a used idle socket, assign to |idle_socket|.  At the end,
 449   //   the |idle_socket_it| will be set to the newest used idle socket.
 450   for (std::list<IdleSocket>::iterator it = idle_sockets->begin();
 451        it != idle_sockets->end();) {
 452     if (!it->IsUsable()) {
 453       DecrementIdleCount();
 454       delete it->socket;
 455       it = idle_sockets->erase(it);
 456       continue;
 457     }
 458
 459     if (it->socket->WasEverUsed()) {
 460       // We found one we can reuse!
 461       idle_socket_it = it;
 462     }
 463
 464     ++it;
 465   }
 466
 467   // If we haven't found an idle socket, that means there are no used idle
 468   // sockets.  Pick the oldest (first) idle socket (FIFO).
 469
 470   if (idle_socket_it == idle_sockets->end() && !idle_sockets->empty())
 471     idle_socket_it = idle_sockets->begin();
 472
 473   if (idle_socket_it != idle_sockets->end()) {
 474     DecrementIdleCount();
 475     base::TimeDelta idle_time =
 476         base::TimeTicks::Now() - idle_socket_it->start_time;
 477     IdleSocket idle_socket = *idle_socket_it;
 478     idle_sockets->erase(idle_socket_it);
 479     // TODO(davidben): If |idle_time| is under some low watermark, consider
 480     // treating as UNUSED rather than UNUSED_IDLE. This will avoid
 481     // HttpNetworkTransaction retrying on some errors.
 482     ClientSocketHandle::SocketReuseType reuse_type =
 483         idle_socket.socket->WasEverUsed() ?
 484             ClientSocketHandle::REUSED_IDLE :
 485             ClientSocketHandle::UNUSED_IDLE;
 486     HandOutSocket(
 487         scoped_ptr<StreamSocket>(idle_socket.socket),
 488         reuse_type,
 489         LoadTimingInfo::ConnectTiming(),
 490         request.handle(),
 491         idle_time,
 492         group,
 493         request.net_log());
 494     return true;
 495   }
 496
 497   return false;
 498 }
 499
 500 // static
 501 void ClientSocketPoolBaseHelper::LogBoundConnectJobToRequest(
 502     const NetLog::Source& connect_job_source, const Request& request) {
 503   request.net_log().AddEvent(NetLog::TYPE_SOCKET_POOL_BOUND_TO_CONNECT_JOB,
 504                              connect_job_source.ToEventParametersCallback());
 505 }
 506
 507 void ClientSocketPoolBaseHelper::CancelRequest(
 508     const std::string& group_name, ClientSocketHandle* handle) {
 509   PendingCallbackMap::iterator callback_it = pending_callback_map_.find(handle);
 510   if (callback_it != pending_callback_map_.end()) {
 511     int result = callback_it->second.result;
 512     pending_callback_map_.erase(callback_it);
 513     scoped_ptr<StreamSocket> socket = handle->PassSocket();
 514     if (socket) {
 515       if (result != OK)
 516         socket->Disconnect();
 517       ReleaseSocket(handle->group_name(), socket.Pass(), handle->id());
 518     }
 519     return;
 520   }
 521
 522   CHECK(ContainsKey(group_map_, group_name));
 523
 524   Group* group = GetOrCreateGroup(group_name);
 525
 526   // Search pending_requests for matching handle.
 527   scoped_ptr<const Request> request =
 528       group->FindAndRemovePendingRequest(handle);
 529   if (request) {
 530     request->net_log().AddEvent(NetLog::TYPE_CANCELLED);
 531     request->net_log().EndEvent(NetLog::TYPE_SOCKET_POOL);
 532
 533     // We let the job run, unless we're at the socket limit and there is
 534     // not another request waiting on the job.
 535     if (group->jobs().size() > group->pending_request_count() &&
 536         ReachedMaxSocketsLimit()) {
 537       RemoveConnectJob(*group->jobs().begin(), group);
 538       CheckForStalledSocketGroups();
 539     }
 540   }
 541 }
 542
 543 bool ClientSocketPoolBaseHelper::HasGroup(const std::string& group_name) const {
 544   return ContainsKey(group_map_, group_name);
 545 }
 546
 547 void ClientSocketPoolBaseHelper::CloseIdleSockets() {
 548   CleanupIdleSockets(true);
 549   DCHECK_EQ(0, idle_socket_count_);
 550 }
 551
 552 int ClientSocketPoolBaseHelper::IdleSocketCountInGroup(
 553     const std::string& group_name) const {
 554   GroupMap::const_iterator i = group_map_.find(group_name);
 555   CHECK(i != group_map_.end());
 556
 557   return i->second->idle_sockets().size();
 558 }
 559
 560 LoadState ClientSocketPoolBaseHelper::GetLoadState(
 561     const std::string& group_name,
 562     const ClientSocketHandle* handle) const {
 563   if (ContainsKey(pending_callback_map_, handle))
 564     return LOAD_STATE_CONNECTING;
 565
 566   if (!ContainsKey(group_map_, group_name)) {
 567     NOTREACHED() << "ClientSocketPool does not contain group: " << group_name
 568                  << " for handle: " << handle;
 569     return LOAD_STATE_IDLE;
 570   }
 571
 572   // Can't use operator[] since it is non-const.
 573   const Group& group = *group_map_.find(group_name)->second;
 574
 575   if (group.HasConnectJobForHandle(handle)) {
 576     // Just return the state  of the farthest along ConnectJob for the first
 577     // group.jobs().size() pending requests.
 578     LoadState max_state = LOAD_STATE_IDLE;
 579     for (ConnectJobSet::const_iterator job_it = group.jobs().begin();
 580          job_it != group.jobs().end(); ++job_it) {
 581       max_state = std::max(max_state, (*job_it)->GetLoadState());
 582     }
 583     return max_state;
 584   }
 585
 586   if (group.IsStalledOnPoolMaxSockets(max_sockets_per_group_))
 587     return LOAD_STATE_WAITING_FOR_STALLED_SOCKET_POOL;
 588   return LOAD_STATE_WAITING_FOR_AVAILABLE_SOCKET;
 589 }
 590
 591 base::DictionaryValue* ClientSocketPoolBaseHelper::GetInfoAsValue(
 592     const std::string& name, const std::string& type) const {
 593   base::DictionaryValue* dict = new base::DictionaryValue();
 594   dict->SetString("name", name);
 595   dict->SetString("type", type);
 596   dict->SetInteger("handed_out_socket_count", handed_out_socket_count_);
 597   dict->SetInteger("connecting_socket_count", connecting_socket_count_);
 598   dict->SetInteger("idle_socket_count", idle_socket_count_);
 599   dict->SetInteger("max_socket_count", max_sockets_);
 600   dict->SetInteger("max_sockets_per_group", max_sockets_per_group_);
 601   dict->SetInteger("pool_generation_number", pool_generation_number_);
 602
 603   if (group_map_.empty())
 604     return dict;
 605
 606   base::DictionaryValue* all_groups_dict = new base::DictionaryValue();
 607   for (GroupMap::const_iterator it = group_map_.begin();
 608        it != group_map_.end(); it++) {
 609     const Group* group = it->second;
 610     base::DictionaryValue* group_dict = new base::DictionaryValue();
 611
 612     group_dict->SetInteger("pending_request_count",
 613                            group->pending_request_count());
 614     if (group->has_pending_requests()) {
 615       group_dict->SetString(
 616           "top_pending_priority",
 617           RequestPriorityToString(group->TopPendingPriority()));
 618     }
 619
 620     group_dict->SetInteger("active_socket_count", group->active_socket_count());
 621
 622     base::ListValue* idle_socket_list = new base::ListValue();
 623     std::list<IdleSocket>::const_iterator idle_socket;
 624     for (idle_socket = group->idle_sockets().begin();
 625          idle_socket != group->idle_sockets().end();
 626          idle_socket++) {
 627       int source_id = idle_socket->socket->NetLog().source().id;
 628       idle_socket_list->Append(new base::FundamentalValue(source_id));
 629     }
 630     group_dict->Set("idle_sockets", idle_socket_list);
 631
 632     base::ListValue* connect_jobs_list = new base::ListValue();
 633     std::set<ConnectJob*>::const_iterator job = group->jobs().begin();
 634     for (job = group->jobs().begin(); job != group->jobs().end(); job++) {
 635       int source_id = (*job)->net_log().source().id;
 636       connect_jobs_list->Append(new base::FundamentalValue(source_id));
 637     }
 638     group_dict->Set("connect_jobs", connect_jobs_list);
 639
 640     group_dict->SetBoolean("is_stalled",
 641                            group->IsStalledOnPoolMaxSockets(
 642                                max_sockets_per_group_));
 643     group_dict->SetBoolean("backup_job_timer_is_running",
 644                            group->BackupJobTimerIsRunning());
 645
 646     all_groups_dict->SetWithoutPathExpansion(it->first, group_dict);
 647   }
 648   dict->Set("groups", all_groups_dict);
 649   return dict;
 650 }
 651
 652 bool ClientSocketPoolBaseHelper::IdleSocket::IsUsable() const {
 653   if (socket->WasEverUsed())
 654     return socket->IsConnectedAndIdle();
 655   return socket->IsConnected();
 656 }
 657
 658 bool ClientSocketPoolBaseHelper::IdleSocket::ShouldCleanup(
 659     base::TimeTicks now,
 660     base::TimeDelta timeout) const {
 661   bool timed_out = (now - start_time) >= timeout;
 662   if (timed_out)
 663     return true;
 664   return !IsUsable();
 665 }
 666
 667 void ClientSocketPoolBaseHelper::CleanupIdleSockets(bool force) {
 668   if (idle_socket_count_ == 0)
 669     return;
 670
 671   // Current time value. Retrieving it once at the function start rather than
 672   // inside the inner loop, since it shouldn't change by any meaningful amount.
 673   base::TimeTicks now = base::TimeTicks::Now();
 674
 675   GroupMap::iterator i = group_map_.begin();
 676   while (i != group_map_.end()) {
 677     Group* group = i->second;
 678
 679     std::list<IdleSocket>::iterator j = group->mutable_idle_sockets()->begin();
 680     while (j != group->idle_sockets().end()) {
 681       base::TimeDelta timeout =
 682           j->socket->WasEverUsed() ?
 683           used_idle_socket_timeout_ : unused_idle_socket_timeout_;
 684       if (force || j->ShouldCleanup(now, timeout)) {
 685         delete j->socket;
 686         j = group->mutable_idle_sockets()->erase(j);
 687         DecrementIdleCount();
 688       } else {
 689         ++j;
 690       }
 691     }
 692
 693     // Delete group if no longer needed.
 694     if (group->IsEmpty()) {
 695       RemoveGroup(i++);
 696     } else {
 697       ++i;
 698     }
 699   }
 700 }
 701
 702 ClientSocketPoolBaseHelper::Group* ClientSocketPoolBaseHelper::GetOrCreateGroup(
 703     const std::string& group_name) {
 704   GroupMap::iterator it = group_map_.find(group_name);
 705   if (it != group_map_.end())
 706     return it->second;
 707   Group* group = new Group;
 708   group_map_[group_name] = group;
 709   return group;
 710 }
 711
 712 void ClientSocketPoolBaseHelper::RemoveGroup(const std::string& group_name) {
 713   GroupMap::iterator it = group_map_.find(group_name);
 714   CHECK(it != group_map_.end());
 715
 716   RemoveGroup(it);
 717 }
 718
 719 void ClientSocketPoolBaseHelper::RemoveGroup(GroupMap::iterator it) {
 720   delete it->second;
 721   group_map_.erase(it);
 722 }
 723
 724 // static
 725 bool ClientSocketPoolBaseHelper::connect_backup_jobs_enabled() {
 726   return g_connect_backup_jobs_enabled;
 727 }
 728
 729 // static
 730 bool ClientSocketPoolBaseHelper::set_connect_backup_jobs_enabled(bool enabled) {
 731   bool old_value = g_connect_backup_jobs_enabled;
 732   g_connect_backup_jobs_enabled = enabled;
 733   return old_value;
 734 }
 735
 736 void ClientSocketPoolBaseHelper::EnableConnectBackupJobs() {
 737   connect_backup_jobs_enabled_ = g_connect_backup_jobs_enabled;
 738 }
 739
 740 void ClientSocketPoolBaseHelper::IncrementIdleCount() {
 741   if (++idle_socket_count_ == 1 && use_cleanup_timer_)
 742     StartIdleSocketTimer();
 743 }
 744
 745 void ClientSocketPoolBaseHelper::DecrementIdleCount() {
 746   if (--idle_socket_count_ == 0)
 747     timer_.Stop();
 748 }
 749
 750 // static
 751 bool ClientSocketPoolBaseHelper::cleanup_timer_enabled() {
 752   return g_cleanup_timer_enabled;
 753 }
 754
 755 // static
 756 bool ClientSocketPoolBaseHelper::set_cleanup_timer_enabled(bool enabled) {
 757   bool old_value = g_cleanup_timer_enabled;
 758   g_cleanup_timer_enabled = enabled;
 759   return old_value;
 760 }
 761
 762 void ClientSocketPoolBaseHelper::StartIdleSocketTimer() {
 763   timer_.Start(FROM_HERE, TimeDelta::FromSeconds(kCleanupInterval), this,
 764                &ClientSocketPoolBaseHelper::OnCleanupTimerFired);
 765 }
 766
 767 void ClientSocketPoolBaseHelper::ReleaseSocket(const std::string& group_name,
 768                                                scoped_ptr<StreamSocket> socket,
 769                                                int id) {
 770   GroupMap::iterator i = group_map_.find(group_name);
 771   CHECK(i != group_map_.end());
 772
 773   Group* group = i->second;
 774
 775   CHECK_GT(handed_out_socket_count_, 0);
 776   handed_out_socket_count_--;
 777
 778   CHECK_GT(group->active_socket_count(), 0);
 779   group->DecrementActiveSocketCount();
 780
 781   const bool can_reuse = socket->IsConnectedAndIdle() &&
 782       id == pool_generation_number_;
 783   if (can_reuse) {
 784     // Add it to the idle list.
 785     AddIdleSocket(socket.Pass(), group);
 786     OnAvailableSocketSlot(group_name, group);
 787   } else {
 788     socket.reset();
 789   }
 790
 791   CheckForStalledSocketGroups();
 792 }
 793
 794 void ClientSocketPoolBaseHelper::CheckForStalledSocketGroups() {
 795   // If we have idle sockets, see if we can give one to the top-stalled group.
 796   std::string top_group_name;
 797   Group* top_group = NULL;
 798   if (!FindTopStalledGroup(&top_group, &top_group_name)) {
 799     // There may still be a stalled group in a lower level pool.
 800     for (std::set<LowerLayeredPool*>::iterator it = lower_pools_.begin();
 801          it != lower_pools_.end();
 802          ++it) {
 803        if ((*it)->IsStalled()) {
 804          CloseOneIdleSocket();
 805          break;
 806        }
 807     }
 808     return;
 809   }
 810
 811   if (ReachedMaxSocketsLimit()) {
 812     if (idle_socket_count() > 0) {
 813       CloseOneIdleSocket();
 814     } else {
 815       // We can't activate more sockets since we're already at our global
 816       // limit.
 817       return;
 818     }
 819   }
 820
 821   // Note:  we don't loop on waking stalled groups.  If the stalled group is at
 822   //        its limit, may be left with other stalled groups that could be
 823   //        woken.  This isn't optimal, but there is no starvation, so to avoid
 824   //        the looping we leave it at this.
 825   OnAvailableSocketSlot(top_group_name, top_group);
 826 }
 827
 828 // Search for the highest priority pending request, amongst the groups that
 829 // are not at the |max_sockets_per_group_| limit. Note: for requests with
 830 // the same priority, the winner is based on group hash ordering (and not
 831 // insertion order).
 832 bool ClientSocketPoolBaseHelper::FindTopStalledGroup(
 833     Group** group,
 834     std::string* group_name) const {
 835   CHECK((group && group_name) || (!group && !group_name));
 836   Group* top_group = NULL;
 837   const std::string* top_group_name = NULL;
 838   bool has_stalled_group = false;
 839   for (GroupMap::const_iterator i = group_map_.begin();
 840        i != group_map_.end(); ++i) {
 841     Group* curr_group = i->second;
 842     if (!curr_group->has_pending_requests())
 843       continue;
 844     if (curr_group->IsStalledOnPoolMaxSockets(max_sockets_per_group_)) {
 845       if (!group)
 846         return true;
 847       has_stalled_group = true;
 848       bool has_higher_priority = !top_group ||
 849           curr_group->TopPendingPriority() > top_group->TopPendingPriority();
 850       if (has_higher_priority) {
 851         top_group = curr_group;
 852         top_group_name = &i->first;
 853       }
 854     }
 855   }
 856
 857   if (top_group) {
 858     CHECK(group);
 859     *group = top_group;
 860     *group_name = *top_group_name;
 861   } else {
 862     CHECK(!has_stalled_group);
 863   }
 864   return has_stalled_group;
 865 }
 866
 867 void ClientSocketPoolBaseHelper::OnConnectJobComplete(
 868     int result, ConnectJob* job) {
 869   // TODO(vadimt): Remove ScopedTracker below once crbug.com/436634 is fixed.
 870   tracked_objects::ScopedTracker tracking_profile(
 871       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 872           "436634 ClientSocketPoolBaseHelper::OnConnectJobComplete"));
 873
 874   DCHECK_NE(ERR_IO_PENDING, result);
 875   const std::string group_name = job->group_name();
 876   GroupMap::iterator group_it = group_map_.find(group_name);
 877   CHECK(group_it != group_map_.end());
 878   Group* group = group_it->second;
 879
 880   scoped_ptr<StreamSocket> socket = job->PassSocket();
 881
 882   // Copies of these are needed because |job| may be deleted before they are
 883   // accessed.
 884   BoundNetLog job_log = job->net_log();
 885   LoadTimingInfo::ConnectTiming connect_timing = job->connect_timing();
 886
 887   // RemoveConnectJob(job, _) must be called by all branches below;
 888   // otherwise, |job| will be leaked.
 889
 890   if (result == OK) {
 891     DCHECK(socket.get());
 892     RemoveConnectJob(job, group);
 893     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 894     if (request) {
 895       LogBoundConnectJobToRequest(job_log.source(), *request);
 896       HandOutSocket(
 897           socket.Pass(), ClientSocketHandle::UNUSED, connect_timing,
 898           request->handle(), base::TimeDelta(), group, request->net_log());
 899       request->net_log().EndEvent(NetLog::TYPE_SOCKET_POOL);
 900       InvokeUserCallbackLater(request->handle(), request->callback(), result);
 901     } else {
 902       AddIdleSocket(socket.Pass(), group);
 903       OnAvailableSocketSlot(group_name, group);
 904       CheckForStalledSocketGroups();
 905     }
 906   } else {
 907     // If we got a socket, it must contain error information so pass that
 908     // up so that the caller can retrieve it.
 909     bool handed_out_socket = false;
 910     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 911     if (request) {
 912       LogBoundConnectJobToRequest(job_log.source(), *request);
 913       job->GetAdditionalErrorState(request->handle());
 914       RemoveConnectJob(job, group);
 915       if (socket.get()) {
 916         handed_out_socket = true;
 917         HandOutSocket(socket.Pass(), ClientSocketHandle::UNUSED,
 918                       connect_timing, request->handle(), base::TimeDelta(),
 919                       group, request->net_log());
 920       }
 921       request->net_log().EndEventWithNetErrorCode(
 922           NetLog::TYPE_SOCKET_POOL, result);
 923       InvokeUserCallbackLater(request->handle(), request->callback(), result);
 924     } else {
 925       RemoveConnectJob(job, group);
 926     }
 927     if (!handed_out_socket) {
 928       OnAvailableSocketSlot(group_name, group);
 929       CheckForStalledSocketGroups();
 930     }
 931   }
 932 }
 933
 934 void ClientSocketPoolBaseHelper::OnIPAddressChanged() {
 935   FlushWithError(ERR_NETWORK_CHANGED);
 936 }
 937
 938 void ClientSocketPoolBaseHelper::FlushWithError(int error) {
 939   pool_generation_number_++;
 940   CancelAllConnectJobs();
 941   CloseIdleSockets();
 942   CancelAllRequestsWithError(error);
 943 }
 944
 945 void ClientSocketPoolBaseHelper::RemoveConnectJob(ConnectJob* job,
 946                                                   Group* group) {
 947   CHECK_GT(connecting_socket_count_, 0);
 948   connecting_socket_count_--;
 949
 950   DCHECK(group);
 951   group->RemoveJob(job);
 952 }
 953
 954 void ClientSocketPoolBaseHelper::OnAvailableSocketSlot(
 955     const std::string& group_name, Group* group) {
 956   DCHECK(ContainsKey(group_map_, group_name));
 957   if (group->IsEmpty()) {
 958     RemoveGroup(group_name);
 959   } else if (group->has_pending_requests()) {
 960     ProcessPendingRequest(group_name, group);
 961   }
 962 }
 963
 964 void ClientSocketPoolBaseHelper::ProcessPendingRequest(
 965     const std::string& group_name, Group* group) {
 966   const Request* next_request = group->GetNextPendingRequest();
 967   DCHECK(next_request);
 968   int rv = RequestSocketInternal(group_name, *next_request);
 969   if (rv != ERR_IO_PENDING) {
 970     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 971     DCHECK(request);
 972     if (group->IsEmpty())
 973       RemoveGroup(group_name);
 974
 975     request->net_log().EndEventWithNetErrorCode(NetLog::TYPE_SOCKET_POOL, rv);
 976     InvokeUserCallbackLater(request->handle(), request->callback(), rv);
 977   }
 978 }
 979
 980 void ClientSocketPoolBaseHelper::HandOutSocket(
 981     scoped_ptr<StreamSocket> socket,
 982     ClientSocketHandle::SocketReuseType reuse_type,
 983     const LoadTimingInfo::ConnectTiming& connect_timing,
 984     ClientSocketHandle* handle,
 985     base::TimeDelta idle_time,
 986     Group* group,
 987     const BoundNetLog& net_log) {
 988   DCHECK(socket);
 989   handle->SetSocket(socket.Pass());
 990   handle->set_reuse_type(reuse_type);
 991   handle->set_idle_time(idle_time);
 992   handle->set_pool_id(pool_generation_number_);
 993   handle->set_connect_timing(connect_timing);
 994
 995   if (handle->is_reused()) {
 996     net_log.AddEvent(
 997         NetLog::TYPE_SOCKET_POOL_REUSED_AN_EXISTING_SOCKET,
 998         NetLog::IntegerCallback(
 999             "idle_ms", static_cast<int>(idle_time.InMilliseconds())));
1000   }
1001
1002   net_log.AddEvent(
1003       NetLog::TYPE_SOCKET_POOL_BOUND_TO_SOCKET,
1004       handle->socket()->NetLog().source().ToEventParametersCallback());
1005
1006   handed_out_socket_count_++;
1007   group->IncrementActiveSocketCount();
1008 }
1009
1010 void ClientSocketPoolBaseHelper::AddIdleSocket(
1011     scoped_ptr<StreamSocket> socket,
1012     Group* group) {
1013   DCHECK(socket);
1014   IdleSocket idle_socket;
1015   idle_socket.socket = socket.release();
1016   idle_socket.start_time = base::TimeTicks::Now();
1017
1018   group->mutable_idle_sockets()->push_back(idle_socket);
1019   IncrementIdleCount();
1020 }
1021
1022 void ClientSocketPoolBaseHelper::CancelAllConnectJobs() {
1023   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end();) {
1024     Group* group = i->second;
1025     connecting_socket_count_ -= group->jobs().size();
1026     group->RemoveAllJobs();
1027
1028     // Delete group if no longer needed.
1029     if (group->IsEmpty()) {
1030       // RemoveGroup() will call .erase() which will invalidate the iterator,
1031       // but i will already have been incremented to a valid iterator before
1032       // RemoveGroup() is called.
1033       RemoveGroup(i++);
1034     } else {
1035       ++i;
1036     }
1037   }
1038   DCHECK_EQ(0, connecting_socket_count_);
1039 }
1040
1041 void ClientSocketPoolBaseHelper::CancelAllRequestsWithError(int error) {
1042   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end();) {
1043     Group* group = i->second;
1044
1045     while (true) {
1046       scoped_ptr<const Request> request = group->PopNextPendingRequest();
1047       if (!request)
1048         break;
1049       InvokeUserCallbackLater(request->handle(), request->callback(), error);
1050     }
1051
1052     // Delete group if no longer needed.
1053     if (group->IsEmpty()) {
1054       // RemoveGroup() will call .erase() which will invalidate the iterator,
1055       // but i will already have been incremented to a valid iterator before
1056       // RemoveGroup() is called.
1057       RemoveGroup(i++);
1058     } else {
1059       ++i;
1060     }
1061   }
1062 }
1063
1064 bool ClientSocketPoolBaseHelper::ReachedMaxSocketsLimit() const {
1065   // Each connecting socket will eventually connect and be handed out.
1066   int total = handed_out_socket_count_ + connecting_socket_count_ +
1067       idle_socket_count();
1068   // There can be more sockets than the limit since some requests can ignore
1069   // the limit
1070   if (total < max_sockets_)
1071     return false;
1072   return true;
1073 }
1074
1075 bool ClientSocketPoolBaseHelper::CloseOneIdleSocket() {
1076   if (idle_socket_count() == 0)
1077     return false;
1078   return CloseOneIdleSocketExceptInGroup(NULL);
1079 }
1080
1081 bool ClientSocketPoolBaseHelper::CloseOneIdleSocketExceptInGroup(
1082     const Group* exception_group) {
1083   CHECK_GT(idle_socket_count(), 0);
1084
1085   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end(); ++i) {
1086     Group* group = i->second;
1087     if (exception_group == group)
1088       continue;
1089     std::list<IdleSocket>* idle_sockets = group->mutable_idle_sockets();
1090
1091     if (!idle_sockets->empty()) {
1092       delete idle_sockets->front().socket;
1093       idle_sockets->pop_front();
1094       DecrementIdleCount();
1095       if (group->IsEmpty())
1096         RemoveGroup(i);
1097
1098       return true;
1099     }
1100   }
1101
1102   return false;
1103 }
1104
1105 bool ClientSocketPoolBaseHelper::CloseOneIdleConnectionInHigherLayeredPool() {
1106   // This pool doesn't have any idle sockets. It's possible that a pool at a
1107   // higher layer is holding one of this sockets active, but it's actually idle.
1108   // Query the higher layers.
1109   for (std::set<HigherLayeredPool*>::const_iterator it = higher_pools_.begin();
1110        it != higher_pools_.end(); ++it) {
1111     if ((*it)->CloseOneIdleConnection())
1112       return true;
1113   }
1114   return false;
1115 }
1116
1117 void ClientSocketPoolBaseHelper::InvokeUserCallbackLater(
1118     ClientSocketHandle* handle, const CompletionCallback& callback, int rv) {
1119   CHECK(!ContainsKey(pending_callback_map_, handle));
1120   pending_callback_map_[handle] = CallbackResultPair(callback, rv);
1121   base::MessageLoop::current()->PostTask(
1122       FROM_HERE,
1123       base::Bind(&ClientSocketPoolBaseHelper::InvokeUserCallback,
1124                  weak_factory_.GetWeakPtr(), handle));
1125 }
1126
1127 void ClientSocketPoolBaseHelper::InvokeUserCallback(
1128     ClientSocketHandle* handle) {
1129   PendingCallbackMap::iterator it = pending_callback_map_.find(handle);
1130
1131   // Exit if the request has already been cancelled.
1132   if (it == pending_callback_map_.end())
1133     return;
1134
1135   CHECK(!handle->is_initialized());
1136   CompletionCallback callback = it->second.callback;
1137   int result = it->second.result;
1138   pending_callback_map_.erase(it);
1139   callback.Run(result);
1140 }
1141
1142 void ClientSocketPoolBaseHelper::TryToCloseSocketsInLayeredPools() {
1143   while (IsStalled()) {
1144     // Closing a socket will result in calling back into |this| to use the freed
1145     // socket slot, so nothing else is needed.
1146     if (!CloseOneIdleConnectionInHigherLayeredPool())
1147       return;
1148   }
1149 }
1150
1151 ClientSocketPoolBaseHelper::Group::Group()
1152     : unassigned_job_count_(0),
1153       pending_requests_(NUM_PRIORITIES),
1154       active_socket_count_(0) {}
1155
1156 ClientSocketPoolBaseHelper::Group::~Group() {
1157   DCHECK_EQ(0u, unassigned_job_count_);
1158 }
1159
1160 void ClientSocketPoolBaseHelper::Group::StartBackupJobTimer(
1161     const std::string& group_name,
1162     ClientSocketPoolBaseHelper* pool) {
1163   // Only allow one timer to run at a time.
1164   if (BackupJobTimerIsRunning())
1165     return;
1166
1167   // Unretained here is okay because |backup_job_timer_| is
1168   // automatically cancelled when it's destroyed.
1169   backup_job_timer_.Start(
1170       FROM_HERE, pool->ConnectRetryInterval(),
1171       base::Bind(&Group::OnBackupJobTimerFired, base::Unretained(this),
1172                  group_name, pool));
1173 }
1174
1175 bool ClientSocketPoolBaseHelper::Group::BackupJobTimerIsRunning() const {
1176   return backup_job_timer_.IsRunning();
1177 }
1178
1179 bool ClientSocketPoolBaseHelper::Group::TryToUseUnassignedConnectJob() {
1180   SanityCheck();
1181
1182   if (unassigned_job_count_ == 0)
1183     return false;
1184   --unassigned_job_count_;
1185   return true;
1186 }
1187
1188 void ClientSocketPoolBaseHelper::Group::AddJob(scoped_ptr<ConnectJob> job,
1189                                                bool is_preconnect) {
1190   SanityCheck();
1191
1192   if (is_preconnect)
1193     ++unassigned_job_count_;
1194   jobs_.insert(job.release());
1195 }
1196
1197 void ClientSocketPoolBaseHelper::Group::RemoveJob(ConnectJob* job) {
1198   scoped_ptr<ConnectJob> owned_job(job);
1199   SanityCheck();
1200
1201   std::set<ConnectJob*>::iterator it = jobs_.find(job);
1202   if (it != jobs_.end()) {
1203     jobs_.erase(it);
1204   } else {
1205     NOTREACHED();
1206   }
1207   size_t job_count = jobs_.size();
1208   if (job_count < unassigned_job_count_)
1209     unassigned_job_count_ = job_count;
1210
1211   // If we've got no more jobs for this group, then we no longer need a
1212   // backup job either.
1213   if (jobs_.empty())
1214     backup_job_timer_.Stop();
1215 }
1216
1217 void ClientSocketPoolBaseHelper::Group::OnBackupJobTimerFired(
1218     std::string group_name,
1219     ClientSocketPoolBaseHelper* pool) {
1220   // If there are no more jobs pending, there is no work to do.
1221   // If we've done our cleanups correctly, this should not happen.
1222   if (jobs_.empty()) {
1223     NOTREACHED();
1224     return;
1225   }
1226
1227   // If our old job is waiting on DNS, or if we can't create any sockets
1228   // right now due to limits, just reset the timer.
1229   if (pool->ReachedMaxSocketsLimit() ||
1230       !HasAvailableSocketSlot(pool->max_sockets_per_group_) ||
1231       (*jobs_.begin())->GetLoadState() == LOAD_STATE_RESOLVING_HOST) {
1232     StartBackupJobTimer(group_name, pool);
1233     return;
1234   }
1235
1236   if (pending_requests_.empty())
1237     return;
1238
1239   scoped_ptr<ConnectJob> backup_job =
1240       pool->connect_job_factory_->NewConnectJob(
1241           group_name, *pending_requests_.FirstMax().value(), pool);
1242   backup_job->net_log().AddEvent(NetLog::TYPE_BACKUP_CONNECT_JOB_CREATED);
1243   SIMPLE_STATS_COUNTER("socket.backup_created");
1244   int rv = backup_job->Connect();
1245   pool->connecting_socket_count_++;
1246   ConnectJob* raw_backup_job = backup_job.get();
1247   AddJob(backup_job.Pass(), false);
1248   if (rv != ERR_IO_PENDING)
1249     pool->OnConnectJobComplete(rv, raw_backup_job);
1250 }
1251
1252 void ClientSocketPoolBaseHelper::Group::SanityCheck() {
1253   DCHECK_LE(unassigned_job_count_, jobs_.size());
1254 }
1255
1256 void ClientSocketPoolBaseHelper::Group::RemoveAllJobs() {
1257   SanityCheck();
1258
1259   // Delete active jobs.
1260   STLDeleteElements(&jobs_);
1261   unassigned_job_count_ = 0;
1262
1263   // Stop backup job timer.
1264   backup_job_timer_.Stop();
1265 }
1266
1267 const ClientSocketPoolBaseHelper::Request*
1268 ClientSocketPoolBaseHelper::Group::GetNextPendingRequest() const {
1269   return
1270       pending_requests_.empty() ? NULL : pending_requests_.FirstMax().value();
1271 }
1272
1273 bool ClientSocketPoolBaseHelper::Group::HasConnectJobForHandle(
1274     const ClientSocketHandle* handle) const {
1275   // Search the first |jobs_.size()| pending requests for |handle|.
1276   // If it's farther back in the deque than that, it doesn't have a
1277   // corresponding ConnectJob.
1278   size_t i = 0;
1279   for (RequestQueue::Pointer pointer = pending_requests_.FirstMax();
1280        !pointer.is_null() && i < jobs_.size();
1281        pointer = pending_requests_.GetNextTowardsLastMin(pointer), ++i) {
1282     if (pointer.value()->handle() == handle)
1283       return true;
1284   }
1285   return false;
1286 }
1287
1288 void ClientSocketPoolBaseHelper::Group::InsertPendingRequest(
1289     scoped_ptr<const Request> request) {
1290   // This value must be cached before we release |request|.
1291   RequestPriority priority = request->priority();
1292   if (request->ignore_limits()) {
1293     // Put requests with ignore_limits == true (which should have
1294     // priority == MAXIMUM_PRIORITY) ahead of other requests with
1295     // MAXIMUM_PRIORITY.
1296     DCHECK_EQ(priority, MAXIMUM_PRIORITY);
1297     pending_requests_.InsertAtFront(request.release(), priority);
1298   } else {
1299     pending_requests_.Insert(request.release(), priority);
1300   }
1301 }
1302
1303 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1304 ClientSocketPoolBaseHelper::Group::PopNextPendingRequest() {
1305   if (pending_requests_.empty())
1306     return scoped_ptr<const ClientSocketPoolBaseHelper::Request>();
1307   return RemovePendingRequest(pending_requests_.FirstMax());
1308 }
1309
1310 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1311 ClientSocketPoolBaseHelper::Group::FindAndRemovePendingRequest(
1312     ClientSocketHandle* handle) {
1313   for (RequestQueue::Pointer pointer = pending_requests_.FirstMax();
1314        !pointer.is_null();
1315        pointer = pending_requests_.GetNextTowardsLastMin(pointer)) {
1316     if (pointer.value()->handle() == handle) {
1317       scoped_ptr<const Request> request = RemovePendingRequest(pointer);
1318       return request.Pass();
1319     }
1320   }
1321   return scoped_ptr<const ClientSocketPoolBaseHelper::Request>();
1322 }
1323
1324 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1325 ClientSocketPoolBaseHelper::Group::RemovePendingRequest(
1326     const RequestQueue::Pointer& pointer) {
1327   scoped_ptr<const Request> request(pointer.value());
1328   pending_requests_.Erase(pointer);
1329   // If there are no more requests, kill the backup timer.
1330   if (pending_requests_.empty())
1331     backup_job_timer_.Stop();
1332   return request.Pass();
1333 }
1334
1335 }  // namespace internal
1336
1337 }  // namespace net