net/socket/client_socket_pool_base.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/socket/client_socket_pool_base.h"
   6
   7 #include "base/compiler_specific.h"
   8 #include "base/format_macros.h"
   9 #include "base/logging.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/profiler/scoped_tracker.h"
  12 #include "base/stl_util.h"
  13 #include "base/strings/string_util.h"
  14 #include "base/time/time.h"
  15 #include "base/values.h"
  16 #include "net/base/net_errors.h"
  17 #include "net/log/net_log.h"
  18
  19 using base::TimeDelta;
  20
  21 namespace net {
  22
  23 namespace {
  24
  25 // Indicate whether we should enable idle socket cleanup timer. When timer is
  26 // disabled, sockets are closed next time a socket request is made.
  27 bool g_cleanup_timer_enabled = true;
  28
  29 // The timeout value, in seconds, used to clean up idle sockets that can't be
  30 // reused.
  31 //
  32 // Note: It's important to close idle sockets that have received data as soon
  33 // as possible because the received data may cause BSOD on Windows XP under
  34 // some conditions.  See http://crbug.com/4606.
  35 const int kCleanupInterval = 10;  // DO NOT INCREASE THIS TIMEOUT.
  36
  37 // Indicate whether or not we should establish a new transport layer connection
  38 // after a certain timeout has passed without receiving an ACK.
  39 bool g_connect_backup_jobs_enabled = true;
  40
  41 }  // namespace
  42
  43 ConnectJob::ConnectJob(const std::string& group_name,
  44                        base::TimeDelta timeout_duration,
  45                        RequestPriority priority,
  46                        Delegate* delegate,
  47                        const BoundNetLog& net_log)
  48     : group_name_(group_name),
  49       timeout_duration_(timeout_duration),
  50       priority_(priority),
  51       delegate_(delegate),
  52       net_log_(net_log),
  53       idle_(true) {
  54   DCHECK(!group_name.empty());
  55   DCHECK(delegate);
  56   net_log.BeginEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB,
  57                      NetLog::StringCallback("group_name", &group_name_));
  58 }
  59
  60 ConnectJob::~ConnectJob() {
  61   net_log().EndEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB);
  62 }
  63
  64 scoped_ptr<StreamSocket> ConnectJob::PassSocket() {
  65   return socket_.Pass();
  66 }
  67
  68 int ConnectJob::Connect() {
  69   if (timeout_duration_ != base::TimeDelta())
  70     timer_.Start(FROM_HERE, timeout_duration_, this, &ConnectJob::OnTimeout);
  71
  72   idle_ = false;
  73
  74   LogConnectStart();
  75
  76   int rv = ConnectInternal();
  77
  78   if (rv != ERR_IO_PENDING) {
  79     LogConnectCompletion(rv);
  80     delegate_ = NULL;
  81   }
  82
  83   return rv;
  84 }
  85
  86 void ConnectJob::SetSocket(scoped_ptr<StreamSocket> socket) {
  87   if (socket) {
  88     net_log().AddEvent(NetLog::TYPE_CONNECT_JOB_SET_SOCKET,
  89                        socket->NetLog().source().ToEventParametersCallback());
  90   }
  91   socket_ = socket.Pass();
  92 }
  93
  94 void ConnectJob::NotifyDelegateOfCompletion(int rv) {
  95   // The delegate will own |this|.
  96   Delegate* delegate = delegate_;
  97   delegate_ = NULL;
  98
  99   LogConnectCompletion(rv);
 100   delegate->OnConnectJobComplete(rv, this);
 101 }
 102
 103 void ConnectJob::ResetTimer(base::TimeDelta remaining_time) {
 104   timer_.Stop();
 105   timer_.Start(FROM_HERE, remaining_time, this, &ConnectJob::OnTimeout);
 106 }
 107
 108 void ConnectJob::LogConnectStart() {
 109   connect_timing_.connect_start = base::TimeTicks::Now();
 110   net_log().BeginEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_CONNECT);
 111 }
 112
 113 void ConnectJob::LogConnectCompletion(int net_error) {
 114   connect_timing_.connect_end = base::TimeTicks::Now();
 115   net_log().EndEventWithNetErrorCode(
 116       NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_CONNECT, net_error);
 117 }
 118
 119 void ConnectJob::OnTimeout() {
 120   // Make sure the socket is NULL before calling into |delegate|.
 121   SetSocket(scoped_ptr<StreamSocket>());
 122
 123   net_log_.AddEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_TIMED_OUT);
 124
 125   NotifyDelegateOfCompletion(ERR_TIMED_OUT);
 126 }
 127
 128 namespace internal {
 129
 130 ClientSocketPoolBaseHelper::Request::Request(
 131     ClientSocketHandle* handle,
 132     const CompletionCallback& callback,
 133     RequestPriority priority,
 134     bool ignore_limits,
 135     Flags flags,
 136     const BoundNetLog& net_log)
 137     : handle_(handle),
 138       callback_(callback),
 139       priority_(priority),
 140       ignore_limits_(ignore_limits),
 141       flags_(flags),
 142       net_log_(net_log) {
 143   if (ignore_limits_)
 144     DCHECK_EQ(priority_, MAXIMUM_PRIORITY);
 145 }
 146
 147 ClientSocketPoolBaseHelper::Request::~Request() {}
 148
 149 ClientSocketPoolBaseHelper::ClientSocketPoolBaseHelper(
 150     HigherLayeredPool* pool,
 151     int max_sockets,
 152     int max_sockets_per_group,
 153     base::TimeDelta unused_idle_socket_timeout,
 154     base::TimeDelta used_idle_socket_timeout,
 155     ConnectJobFactory* connect_job_factory)
 156     : idle_socket_count_(0),
 157       connecting_socket_count_(0),
 158       handed_out_socket_count_(0),
 159       max_sockets_(max_sockets),
 160       max_sockets_per_group_(max_sockets_per_group),
 161       use_cleanup_timer_(g_cleanup_timer_enabled),
 162       unused_idle_socket_timeout_(unused_idle_socket_timeout),
 163       used_idle_socket_timeout_(used_idle_socket_timeout),
 164       connect_job_factory_(connect_job_factory),
 165       connect_backup_jobs_enabled_(false),
 166       pool_generation_number_(0),
 167       pool_(pool),
 168       weak_factory_(this) {
 169   DCHECK_LE(0, max_sockets_per_group);
 170   DCHECK_LE(max_sockets_per_group, max_sockets);
 171
 172   NetworkChangeNotifier::AddIPAddressObserver(this);
 173 }
 174
 175 ClientSocketPoolBaseHelper::~ClientSocketPoolBaseHelper() {
 176   // Clean up any idle sockets and pending connect jobs.  Assert that we have no
 177   // remaining active sockets or pending requests.  They should have all been
 178   // cleaned up prior to |this| being destroyed.
 179   FlushWithError(ERR_ABORTED);
 180   DCHECK(group_map_.empty());
 181   DCHECK(pending_callback_map_.empty());
 182   DCHECK_EQ(0, connecting_socket_count_);
 183   CHECK(higher_pools_.empty());
 184
 185   NetworkChangeNotifier::RemoveIPAddressObserver(this);
 186
 187   // Remove from lower layer pools.
 188   for (std::set<LowerLayeredPool*>::iterator it = lower_pools_.begin();
 189        it != lower_pools_.end();
 190        ++it) {
 191     (*it)->RemoveHigherLayeredPool(pool_);
 192   }
 193 }
 194
 195 ClientSocketPoolBaseHelper::CallbackResultPair::CallbackResultPair()
 196     : result(OK) {
 197 }
 198
 199 ClientSocketPoolBaseHelper::CallbackResultPair::CallbackResultPair(
 200     const CompletionCallback& callback_in, int result_in)
 201     : callback(callback_in),
 202       result(result_in) {
 203 }
 204
 205 ClientSocketPoolBaseHelper::CallbackResultPair::~CallbackResultPair() {}
 206
 207 bool ClientSocketPoolBaseHelper::IsStalled() const {
 208   // If a lower layer pool is stalled, consider |this| stalled as well.
 209   for (std::set<LowerLayeredPool*>::const_iterator it = lower_pools_.begin();
 210        it != lower_pools_.end();
 211        ++it) {
 212     if ((*it)->IsStalled())
 213       return true;
 214   }
 215
 216   // If fewer than |max_sockets_| are in use, then clearly |this| is not
 217   // stalled.
 218   if ((handed_out_socket_count_ + connecting_socket_count_) < max_sockets_)
 219     return false;
 220   // So in order to be stalled, |this| must be using at least |max_sockets_| AND
 221   // |this| must have a request that is actually stalled on the global socket
 222   // limit.  To find such a request, look for a group that has more requests
 223   // than jobs AND where the number of sockets is less than
 224   // |max_sockets_per_group_|.  (If the number of sockets is equal to
 225   // |max_sockets_per_group_|, then the request is stalled on the group limit,
 226   // which does not count.)
 227   for (GroupMap::const_iterator it = group_map_.begin();
 228        it != group_map_.end(); ++it) {
 229     if (it->second->CanUseAdditionalSocketSlot(max_sockets_per_group_))
 230       return true;
 231   }
 232   return false;
 233 }
 234
 235 void ClientSocketPoolBaseHelper::AddLowerLayeredPool(
 236     LowerLayeredPool* lower_pool) {
 237   DCHECK(pool_);
 238   CHECK(!ContainsKey(lower_pools_, lower_pool));
 239   lower_pools_.insert(lower_pool);
 240   lower_pool->AddHigherLayeredPool(pool_);
 241 }
 242
 243 void ClientSocketPoolBaseHelper::AddHigherLayeredPool(
 244     HigherLayeredPool* higher_pool) {
 245   CHECK(higher_pool);
 246   CHECK(!ContainsKey(higher_pools_, higher_pool));
 247   higher_pools_.insert(higher_pool);
 248 }
 249
 250 void ClientSocketPoolBaseHelper::RemoveHigherLayeredPool(
 251     HigherLayeredPool* higher_pool) {
 252   CHECK(higher_pool);
 253   CHECK(ContainsKey(higher_pools_, higher_pool));
 254   higher_pools_.erase(higher_pool);
 255 }
 256
 257 int ClientSocketPoolBaseHelper::RequestSocket(
 258     const std::string& group_name,
 259     scoped_ptr<const Request> request) {
 260   CHECK(!request->callback().is_null());
 261   CHECK(request->handle());
 262
 263   // Cleanup any timed-out idle sockets if no timer is used.
 264   if (!use_cleanup_timer_)
 265     CleanupIdleSockets(false);
 266
 267   request->net_log().BeginEvent(NetLog::TYPE_SOCKET_POOL);
 268   Group* group = GetOrCreateGroup(group_name);
 269
 270   int rv = RequestSocketInternal(group_name, *request);
 271   if (rv != ERR_IO_PENDING) {
 272     request->net_log().EndEventWithNetErrorCode(NetLog::TYPE_SOCKET_POOL, rv);
 273     CHECK(!request->handle()->is_initialized());
 274     request.reset();
 275   } else {
 276     group->InsertPendingRequest(request.Pass());
 277     // Have to do this asynchronously, as closing sockets in higher level pools
 278     // call back in to |this|, which will cause all sorts of fun and exciting
 279     // re-entrancy issues if the socket pool is doing something else at the
 280     // time.
 281     if (group->CanUseAdditionalSocketSlot(max_sockets_per_group_)) {
 282       base::MessageLoop::current()->PostTask(
 283           FROM_HERE,
 284           base::Bind(
 285               &ClientSocketPoolBaseHelper::TryToCloseSocketsInLayeredPools,
 286               weak_factory_.GetWeakPtr()));
 287     }
 288   }
 289   return rv;
 290 }
 291
 292 void ClientSocketPoolBaseHelper::RequestSockets(
 293     const std::string& group_name,
 294     const Request& request,
 295     int num_sockets) {
 296   DCHECK(request.callback().is_null());
 297   DCHECK(!request.handle());
 298
 299   // Cleanup any timed out idle sockets if no timer is used.
 300   if (!use_cleanup_timer_)
 301     CleanupIdleSockets(false);
 302
 303   if (num_sockets > max_sockets_per_group_) {
 304     num_sockets = max_sockets_per_group_;
 305   }
 306
 307   request.net_log().BeginEvent(
 308       NetLog::TYPE_SOCKET_POOL_CONNECTING_N_SOCKETS,
 309       NetLog::IntegerCallback("num_sockets", num_sockets));
 310
 311   Group* group = GetOrCreateGroup(group_name);
 312
 313   // RequestSocketsInternal() may delete the group.
 314   bool deleted_group = false;
 315
 316   int rv = OK;
 317   for (int num_iterations_left = num_sockets;
 318        group->NumActiveSocketSlots() < num_sockets &&
 319        num_iterations_left > 0 ; num_iterations_left--) {
 320     rv = RequestSocketInternal(group_name, request);
 321     if (rv < 0 && rv != ERR_IO_PENDING) {
 322       // We're encountering a synchronous error.  Give up.
 323       if (!ContainsKey(group_map_, group_name))
 324         deleted_group = true;
 325       break;
 326     }
 327     if (!ContainsKey(group_map_, group_name)) {
 328       // Unexpected.  The group should only be getting deleted on synchronous
 329       // error.
 330       NOTREACHED();
 331       deleted_group = true;
 332       break;
 333     }
 334   }
 335
 336   if (!deleted_group && group->IsEmpty())
 337     RemoveGroup(group_name);
 338
 339   if (rv == ERR_IO_PENDING)
 340     rv = OK;
 341   request.net_log().EndEventWithNetErrorCode(
 342       NetLog::TYPE_SOCKET_POOL_CONNECTING_N_SOCKETS, rv);
 343 }
 344
 345 int ClientSocketPoolBaseHelper::RequestSocketInternal(
 346     const std::string& group_name,
 347     const Request& request) {
 348   ClientSocketHandle* const handle = request.handle();
 349   const bool preconnecting = !handle;
 350   Group* group = GetOrCreateGroup(group_name);
 351
 352   if (!(request.flags() & NO_IDLE_SOCKETS)) {
 353     // Try to reuse a socket.
 354     if (AssignIdleSocketToRequest(request, group))
 355       return OK;
 356   }
 357
 358   // If there are more ConnectJobs than pending requests, don't need to do
 359   // anything.  Can just wait for the extra job to connect, and then assign it
 360   // to the request.
 361   if (!preconnecting && group->TryToUseUnassignedConnectJob())
 362     return ERR_IO_PENDING;
 363
 364   // Can we make another active socket now?
 365   if (!group->HasAvailableSocketSlot(max_sockets_per_group_) &&
 366       !request.ignore_limits()) {
 367     // TODO(willchan): Consider whether or not we need to close a socket in a
 368     // higher layered group. I don't think this makes sense since we would just
 369     // reuse that socket then if we needed one and wouldn't make it down to this
 370     // layer.
 371     request.net_log().AddEvent(
 372         NetLog::TYPE_SOCKET_POOL_STALLED_MAX_SOCKETS_PER_GROUP);
 373     return ERR_IO_PENDING;
 374   }
 375
 376   if (ReachedMaxSocketsLimit() && !request.ignore_limits()) {
 377     // NOTE(mmenke):  Wonder if we really need different code for each case
 378     // here.  Only reason for them now seems to be preconnects.
 379     if (idle_socket_count() > 0) {
 380       // There's an idle socket in this pool. Either that's because there's
 381       // still one in this group, but we got here due to preconnecting bypassing
 382       // idle sockets, or because there's an idle socket in another group.
 383       bool closed = CloseOneIdleSocketExceptInGroup(group);
 384       if (preconnecting && !closed)
 385         return ERR_PRECONNECT_MAX_SOCKET_LIMIT;
 386     } else {
 387       // We could check if we really have a stalled group here, but it requires
 388       // a scan of all groups, so just flip a flag here, and do the check later.
 389       request.net_log().AddEvent(NetLog::TYPE_SOCKET_POOL_STALLED_MAX_SOCKETS);
 390       return ERR_IO_PENDING;
 391     }
 392   }
 393
 394   // We couldn't find a socket to reuse, and there's space to allocate one,
 395   // so allocate and connect a new one.
 396   scoped_ptr<ConnectJob> connect_job(
 397       connect_job_factory_->NewConnectJob(group_name, request, this));
 398
 399   int rv = connect_job->Connect();
 400   if (rv == OK) {
 401     LogBoundConnectJobToRequest(connect_job->net_log().source(), request);
 402     if (!preconnecting) {
 403       HandOutSocket(connect_job->PassSocket(), ClientSocketHandle::UNUSED,
 404                     connect_job->connect_timing(), handle, base::TimeDelta(),
 405                     group, request.net_log());
 406     } else {
 407       AddIdleSocket(connect_job->PassSocket(), group);
 408     }
 409   } else if (rv == ERR_IO_PENDING) {
 410     // If we don't have any sockets in this group, set a timer for potentially
 411     // creating a new one.  If the SYN is lost, this backup socket may complete
 412     // before the slow socket, improving end user latency.
 413     if (connect_backup_jobs_enabled_ && group->IsEmpty()) {
 414       group->StartBackupJobTimer(group_name, this);
 415     }
 416
 417     connecting_socket_count_++;
 418
 419     group->AddJob(connect_job.Pass(), preconnecting);
 420   } else {
 421     LogBoundConnectJobToRequest(connect_job->net_log().source(), request);
 422     scoped_ptr<StreamSocket> error_socket;
 423     if (!preconnecting) {
 424       DCHECK(handle);
 425       connect_job->GetAdditionalErrorState(handle);
 426       error_socket = connect_job->PassSocket();
 427     }
 428     if (error_socket) {
 429       HandOutSocket(error_socket.Pass(), ClientSocketHandle::UNUSED,
 430                     connect_job->connect_timing(), handle, base::TimeDelta(),
 431                     group, request.net_log());
 432     } else if (group->IsEmpty()) {
 433       RemoveGroup(group_name);
 434     }
 435   }
 436
 437   return rv;
 438 }
 439
 440 bool ClientSocketPoolBaseHelper::AssignIdleSocketToRequest(
 441     const Request& request, Group* group) {
 442   std::list<IdleSocket>* idle_sockets = group->mutable_idle_sockets();
 443   std::list<IdleSocket>::iterator idle_socket_it = idle_sockets->end();
 444
 445   // Iterate through the idle sockets forwards (oldest to newest)
 446   //   * Delete any disconnected ones.
 447   //   * If we find a used idle socket, assign to |idle_socket|.  At the end,
 448   //   the |idle_socket_it| will be set to the newest used idle socket.
 449   for (std::list<IdleSocket>::iterator it = idle_sockets->begin();
 450        it != idle_sockets->end();) {
 451     if (!it->IsUsable()) {
 452       DecrementIdleCount();
 453       delete it->socket;
 454       it = idle_sockets->erase(it);
 455       continue;
 456     }
 457
 458     if (it->socket->WasEverUsed()) {
 459       // We found one we can reuse!
 460       idle_socket_it = it;
 461     }
 462
 463     ++it;
 464   }
 465
 466   // If we haven't found an idle socket, that means there are no used idle
 467   // sockets.  Pick the oldest (first) idle socket (FIFO).
 468
 469   if (idle_socket_it == idle_sockets->end() && !idle_sockets->empty())
 470     idle_socket_it = idle_sockets->begin();
 471
 472   if (idle_socket_it != idle_sockets->end()) {
 473     DecrementIdleCount();
 474     base::TimeDelta idle_time =
 475         base::TimeTicks::Now() - idle_socket_it->start_time;
 476     IdleSocket idle_socket = *idle_socket_it;
 477     idle_sockets->erase(idle_socket_it);
 478     // TODO(davidben): If |idle_time| is under some low watermark, consider
 479     // treating as UNUSED rather than UNUSED_IDLE. This will avoid
 480     // HttpNetworkTransaction retrying on some errors.
 481     ClientSocketHandle::SocketReuseType reuse_type =
 482         idle_socket.socket->WasEverUsed() ?
 483             ClientSocketHandle::REUSED_IDLE :
 484             ClientSocketHandle::UNUSED_IDLE;
 485     HandOutSocket(
 486         scoped_ptr<StreamSocket>(idle_socket.socket),
 487         reuse_type,
 488         LoadTimingInfo::ConnectTiming(),
 489         request.handle(),
 490         idle_time,
 491         group,
 492         request.net_log());
 493     return true;
 494   }
 495
 496   return false;
 497 }
 498
 499 // static
 500 void ClientSocketPoolBaseHelper::LogBoundConnectJobToRequest(
 501     const NetLog::Source& connect_job_source, const Request& request) {
 502   request.net_log().AddEvent(NetLog::TYPE_SOCKET_POOL_BOUND_TO_CONNECT_JOB,
 503                              connect_job_source.ToEventParametersCallback());
 504 }
 505
 506 void ClientSocketPoolBaseHelper::CancelRequest(
 507     const std::string& group_name, ClientSocketHandle* handle) {
 508   PendingCallbackMap::iterator callback_it = pending_callback_map_.find(handle);
 509   if (callback_it != pending_callback_map_.end()) {
 510     int result = callback_it->second.result;
 511     pending_callback_map_.erase(callback_it);
 512     scoped_ptr<StreamSocket> socket = handle->PassSocket();
 513     if (socket) {
 514       if (result != OK)
 515         socket->Disconnect();
 516       ReleaseSocket(handle->group_name(), socket.Pass(), handle->id());
 517     }
 518     return;
 519   }
 520
 521   CHECK(ContainsKey(group_map_, group_name));
 522
 523   Group* group = GetOrCreateGroup(group_name);
 524
 525   // Search pending_requests for matching handle.
 526   scoped_ptr<const Request> request =
 527       group->FindAndRemovePendingRequest(handle);
 528   if (request) {
 529     request->net_log().AddEvent(NetLog::TYPE_CANCELLED);
 530     request->net_log().EndEvent(NetLog::TYPE_SOCKET_POOL);
 531
 532     // We let the job run, unless we're at the socket limit and there is
 533     // not another request waiting on the job.
 534     if (group->jobs().size() > group->pending_request_count() &&
 535         ReachedMaxSocketsLimit()) {
 536       RemoveConnectJob(*group->jobs().begin(), group);
 537       CheckForStalledSocketGroups();
 538     }
 539   }
 540 }
 541
 542 bool ClientSocketPoolBaseHelper::HasGroup(const std::string& group_name) const {
 543   return ContainsKey(group_map_, group_name);
 544 }
 545
 546 void ClientSocketPoolBaseHelper::CloseIdleSockets() {
 547   CleanupIdleSockets(true);
 548   DCHECK_EQ(0, idle_socket_count_);
 549 }
 550
 551 int ClientSocketPoolBaseHelper::IdleSocketCountInGroup(
 552     const std::string& group_name) const {
 553   GroupMap::const_iterator i = group_map_.find(group_name);
 554   CHECK(i != group_map_.end());
 555
 556   return i->second->idle_sockets().size();
 557 }
 558
 559 LoadState ClientSocketPoolBaseHelper::GetLoadState(
 560     const std::string& group_name,
 561     const ClientSocketHandle* handle) const {
 562   if (ContainsKey(pending_callback_map_, handle))
 563     return LOAD_STATE_CONNECTING;
 564
 565   GroupMap::const_iterator group_it = group_map_.find(group_name);
 566   if (group_it == group_map_.end()) {
 567     // TODO(mmenke):  This is actually reached in the wild, for unknown reasons.
 568     // Would be great to understand why, and if it's a bug, fix it.  If not,
 569     // should have a test for that case.
 570     NOTREACHED();
 571     return LOAD_STATE_IDLE;
 572   }
 573
 574   const Group& group = *group_it->second;
 575   if (group.HasConnectJobForHandle(handle)) {
 576     // Just return the state of the farthest along ConnectJob for the first
 577     // group.jobs().size() pending requests.
 578     LoadState max_state = LOAD_STATE_IDLE;
 579     for (const auto& job : group.jobs()) {
 580       max_state = std::max(max_state, job->GetLoadState());
 581     }
 582     return max_state;
 583   }
 584
 585   if (group.CanUseAdditionalSocketSlot(max_sockets_per_group_))
 586     return LOAD_STATE_WAITING_FOR_STALLED_SOCKET_POOL;
 587   return LOAD_STATE_WAITING_FOR_AVAILABLE_SOCKET;
 588 }
 589
 590 base::DictionaryValue* ClientSocketPoolBaseHelper::GetInfoAsValue(
 591     const std::string& name, const std::string& type) const {
 592   base::DictionaryValue* dict = new base::DictionaryValue();
 593   dict->SetString("name", name);
 594   dict->SetString("type", type);
 595   dict->SetInteger("handed_out_socket_count", handed_out_socket_count_);
 596   dict->SetInteger("connecting_socket_count", connecting_socket_count_);
 597   dict->SetInteger("idle_socket_count", idle_socket_count_);
 598   dict->SetInteger("max_socket_count", max_sockets_);
 599   dict->SetInteger("max_sockets_per_group", max_sockets_per_group_);
 600   dict->SetInteger("pool_generation_number", pool_generation_number_);
 601
 602   if (group_map_.empty())
 603     return dict;
 604
 605   base::DictionaryValue* all_groups_dict = new base::DictionaryValue();
 606   for (GroupMap::const_iterator it = group_map_.begin();
 607        it != group_map_.end(); it++) {
 608     const Group* group = it->second;
 609     base::DictionaryValue* group_dict = new base::DictionaryValue();
 610
 611     group_dict->SetInteger("pending_request_count",
 612                            group->pending_request_count());
 613     if (group->has_pending_requests()) {
 614       group_dict->SetString(
 615           "top_pending_priority",
 616           RequestPriorityToString(group->TopPendingPriority()));
 617     }
 618
 619     group_dict->SetInteger("active_socket_count", group->active_socket_count());
 620
 621     base::ListValue* idle_socket_list = new base::ListValue();
 622     std::list<IdleSocket>::const_iterator idle_socket;
 623     for (idle_socket = group->idle_sockets().begin();
 624          idle_socket != group->idle_sockets().end();
 625          idle_socket++) {
 626       int source_id = idle_socket->socket->NetLog().source().id;
 627       idle_socket_list->Append(new base::FundamentalValue(source_id));
 628     }
 629     group_dict->Set("idle_sockets", idle_socket_list);
 630
 631     base::ListValue* connect_jobs_list = new base::ListValue();
 632     std::set<ConnectJob*>::const_iterator job = group->jobs().begin();
 633     for (job = group->jobs().begin(); job != group->jobs().end(); job++) {
 634       int source_id = (*job)->net_log().source().id;
 635       connect_jobs_list->Append(new base::FundamentalValue(source_id));
 636     }
 637     group_dict->Set("connect_jobs", connect_jobs_list);
 638
 639     group_dict->SetBoolean("is_stalled", group->CanUseAdditionalSocketSlot(
 640                                              max_sockets_per_group_));
 641     group_dict->SetBoolean("backup_job_timer_is_running",
 642                            group->BackupJobTimerIsRunning());
 643
 644     all_groups_dict->SetWithoutPathExpansion(it->first, group_dict);
 645   }
 646   dict->Set("groups", all_groups_dict);
 647   return dict;
 648 }
 649
 650 bool ClientSocketPoolBaseHelper::IdleSocket::IsUsable() const {
 651   if (socket->WasEverUsed())
 652     return socket->IsConnectedAndIdle();
 653   return socket->IsConnected();
 654 }
 655
 656 bool ClientSocketPoolBaseHelper::IdleSocket::ShouldCleanup(
 657     base::TimeTicks now,
 658     base::TimeDelta timeout) const {
 659   bool timed_out = (now - start_time) >= timeout;
 660   if (timed_out)
 661     return true;
 662   return !IsUsable();
 663 }
 664
 665 void ClientSocketPoolBaseHelper::CleanupIdleSockets(bool force) {
 666   if (idle_socket_count_ == 0)
 667     return;
 668
 669   // Current time value. Retrieving it once at the function start rather than
 670   // inside the inner loop, since it shouldn't change by any meaningful amount.
 671   base::TimeTicks now = base::TimeTicks::Now();
 672
 673   GroupMap::iterator i = group_map_.begin();
 674   while (i != group_map_.end()) {
 675     Group* group = i->second;
 676
 677     std::list<IdleSocket>::iterator j = group->mutable_idle_sockets()->begin();
 678     while (j != group->idle_sockets().end()) {
 679       base::TimeDelta timeout =
 680           j->socket->WasEverUsed() ?
 681           used_idle_socket_timeout_ : unused_idle_socket_timeout_;
 682       if (force || j->ShouldCleanup(now, timeout)) {
 683         delete j->socket;
 684         j = group->mutable_idle_sockets()->erase(j);
 685         DecrementIdleCount();
 686       } else {
 687         ++j;
 688       }
 689     }
 690
 691     // Delete group if no longer needed.
 692     if (group->IsEmpty()) {
 693       RemoveGroup(i++);
 694     } else {
 695       ++i;
 696     }
 697   }
 698 }
 699
 700 ClientSocketPoolBaseHelper::Group* ClientSocketPoolBaseHelper::GetOrCreateGroup(
 701     const std::string& group_name) {
 702   GroupMap::iterator it = group_map_.find(group_name);
 703   if (it != group_map_.end())
 704     return it->second;
 705   Group* group = new Group;
 706   group_map_[group_name] = group;
 707   return group;
 708 }
 709
 710 void ClientSocketPoolBaseHelper::RemoveGroup(const std::string& group_name) {
 711   GroupMap::iterator it = group_map_.find(group_name);
 712   CHECK(it != group_map_.end());
 713
 714   RemoveGroup(it);
 715 }
 716
 717 void ClientSocketPoolBaseHelper::RemoveGroup(GroupMap::iterator it) {
 718   delete it->second;
 719   group_map_.erase(it);
 720 }
 721
 722 // static
 723 bool ClientSocketPoolBaseHelper::connect_backup_jobs_enabled() {
 724   return g_connect_backup_jobs_enabled;
 725 }
 726
 727 // static
 728 bool ClientSocketPoolBaseHelper::set_connect_backup_jobs_enabled(bool enabled) {
 729   bool old_value = g_connect_backup_jobs_enabled;
 730   g_connect_backup_jobs_enabled = enabled;
 731   return old_value;
 732 }
 733
 734 void ClientSocketPoolBaseHelper::EnableConnectBackupJobs() {
 735   connect_backup_jobs_enabled_ = g_connect_backup_jobs_enabled;
 736 }
 737
 738 void ClientSocketPoolBaseHelper::IncrementIdleCount() {
 739   if (++idle_socket_count_ == 1 && use_cleanup_timer_)
 740     StartIdleSocketTimer();
 741 }
 742
 743 void ClientSocketPoolBaseHelper::DecrementIdleCount() {
 744   if (--idle_socket_count_ == 0)
 745     timer_.Stop();
 746 }
 747
 748 // static
 749 bool ClientSocketPoolBaseHelper::cleanup_timer_enabled() {
 750   return g_cleanup_timer_enabled;
 751 }
 752
 753 // static
 754 bool ClientSocketPoolBaseHelper::set_cleanup_timer_enabled(bool enabled) {
 755   bool old_value = g_cleanup_timer_enabled;
 756   g_cleanup_timer_enabled = enabled;
 757   return old_value;
 758 }
 759
 760 void ClientSocketPoolBaseHelper::StartIdleSocketTimer() {
 761   timer_.Start(FROM_HERE, TimeDelta::FromSeconds(kCleanupInterval), this,
 762                &ClientSocketPoolBaseHelper::OnCleanupTimerFired);
 763 }
 764
 765 void ClientSocketPoolBaseHelper::ReleaseSocket(const std::string& group_name,
 766                                                scoped_ptr<StreamSocket> socket,
 767                                                int id) {
 768   GroupMap::iterator i = group_map_.find(group_name);
 769   CHECK(i != group_map_.end());
 770
 771   Group* group = i->second;
 772
 773   CHECK_GT(handed_out_socket_count_, 0);
 774   handed_out_socket_count_--;
 775
 776   CHECK_GT(group->active_socket_count(), 0);
 777   group->DecrementActiveSocketCount();
 778
 779   const bool can_reuse = socket->IsConnectedAndIdle() &&
 780       id == pool_generation_number_;
 781   if (can_reuse) {
 782     // Add it to the idle list.
 783     AddIdleSocket(socket.Pass(), group);
 784     OnAvailableSocketSlot(group_name, group);
 785   } else {
 786     socket.reset();
 787   }
 788
 789   CheckForStalledSocketGroups();
 790 }
 791
 792 void ClientSocketPoolBaseHelper::CheckForStalledSocketGroups() {
 793   // If we have idle sockets, see if we can give one to the top-stalled group.
 794   std::string top_group_name;
 795   Group* top_group = NULL;
 796   if (!FindTopStalledGroup(&top_group, &top_group_name)) {
 797     // There may still be a stalled group in a lower level pool.
 798     for (std::set<LowerLayeredPool*>::iterator it = lower_pools_.begin();
 799          it != lower_pools_.end();
 800          ++it) {
 801        if ((*it)->IsStalled()) {
 802          CloseOneIdleSocket();
 803          break;
 804        }
 805     }
 806     return;
 807   }
 808
 809   if (ReachedMaxSocketsLimit()) {
 810     if (idle_socket_count() > 0) {
 811       CloseOneIdleSocket();
 812     } else {
 813       // We can't activate more sockets since we're already at our global
 814       // limit.
 815       return;
 816     }
 817   }
 818
 819   // Note:  we don't loop on waking stalled groups.  If the stalled group is at
 820   //        its limit, may be left with other stalled groups that could be
 821   //        woken.  This isn't optimal, but there is no starvation, so to avoid
 822   //        the looping we leave it at this.
 823   OnAvailableSocketSlot(top_group_name, top_group);
 824 }
 825
 826 // Search for the highest priority pending request, amongst the groups that
 827 // are not at the |max_sockets_per_group_| limit. Note: for requests with
 828 // the same priority, the winner is based on group hash ordering (and not
 829 // insertion order).
 830 bool ClientSocketPoolBaseHelper::FindTopStalledGroup(
 831     Group** group,
 832     std::string* group_name) const {
 833   CHECK((group && group_name) || (!group && !group_name));
 834   Group* top_group = NULL;
 835   const std::string* top_group_name = NULL;
 836   bool has_stalled_group = false;
 837   for (GroupMap::const_iterator i = group_map_.begin();
 838        i != group_map_.end(); ++i) {
 839     Group* curr_group = i->second;
 840     if (!curr_group->has_pending_requests())
 841       continue;
 842     if (curr_group->CanUseAdditionalSocketSlot(max_sockets_per_group_)) {
 843       if (!group)
 844         return true;
 845       has_stalled_group = true;
 846       bool has_higher_priority = !top_group ||
 847           curr_group->TopPendingPriority() > top_group->TopPendingPriority();
 848       if (has_higher_priority) {
 849         top_group = curr_group;
 850         top_group_name = &i->first;
 851       }
 852     }
 853   }
 854
 855   if (top_group) {
 856     CHECK(group);
 857     *group = top_group;
 858     *group_name = *top_group_name;
 859   } else {
 860     CHECK(!has_stalled_group);
 861   }
 862   return has_stalled_group;
 863 }
 864
 865 void ClientSocketPoolBaseHelper::OnConnectJobComplete(
 866     int result, ConnectJob* job) {
 867   // TODO(vadimt): Remove ScopedTracker below once crbug.com/436634 is fixed.
 868   tracked_objects::ScopedTracker tracking_profile(
 869       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 870           "436634 ClientSocketPoolBaseHelper::OnConnectJobComplete"));
 871
 872   DCHECK_NE(ERR_IO_PENDING, result);
 873   const std::string group_name = job->group_name();
 874   GroupMap::iterator group_it = group_map_.find(group_name);
 875   CHECK(group_it != group_map_.end());
 876   Group* group = group_it->second;
 877
 878   scoped_ptr<StreamSocket> socket = job->PassSocket();
 879
 880   // Copies of these are needed because |job| may be deleted before they are
 881   // accessed.
 882   BoundNetLog job_log = job->net_log();
 883   LoadTimingInfo::ConnectTiming connect_timing = job->connect_timing();
 884
 885   // RemoveConnectJob(job, _) must be called by all branches below;
 886   // otherwise, |job| will be leaked.
 887
 888   if (result == OK) {
 889     DCHECK(socket.get());
 890     RemoveConnectJob(job, group);
 891     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 892     if (request) {
 893       LogBoundConnectJobToRequest(job_log.source(), *request);
 894       HandOutSocket(
 895           socket.Pass(), ClientSocketHandle::UNUSED, connect_timing,
 896           request->handle(), base::TimeDelta(), group, request->net_log());
 897       request->net_log().EndEvent(NetLog::TYPE_SOCKET_POOL);
 898       InvokeUserCallbackLater(request->handle(), request->callback(), result);
 899     } else {
 900       AddIdleSocket(socket.Pass(), group);
 901       OnAvailableSocketSlot(group_name, group);
 902       CheckForStalledSocketGroups();
 903     }
 904   } else {
 905     // If we got a socket, it must contain error information so pass that
 906     // up so that the caller can retrieve it.
 907     bool handed_out_socket = false;
 908     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 909     if (request) {
 910       LogBoundConnectJobToRequest(job_log.source(), *request);
 911       job->GetAdditionalErrorState(request->handle());
 912       RemoveConnectJob(job, group);
 913       if (socket.get()) {
 914         handed_out_socket = true;
 915         HandOutSocket(socket.Pass(), ClientSocketHandle::UNUSED,
 916                       connect_timing, request->handle(), base::TimeDelta(),
 917                       group, request->net_log());
 918       }
 919       request->net_log().EndEventWithNetErrorCode(
 920           NetLog::TYPE_SOCKET_POOL, result);
 921       InvokeUserCallbackLater(request->handle(), request->callback(), result);
 922     } else {
 923       RemoveConnectJob(job, group);
 924     }
 925     if (!handed_out_socket) {
 926       OnAvailableSocketSlot(group_name, group);
 927       CheckForStalledSocketGroups();
 928     }
 929   }
 930 }
 931
 932 void ClientSocketPoolBaseHelper::OnIPAddressChanged() {
 933   FlushWithError(ERR_NETWORK_CHANGED);
 934 }
 935
 936 void ClientSocketPoolBaseHelper::FlushWithError(int error) {
 937   pool_generation_number_++;
 938   CancelAllConnectJobs();
 939   CloseIdleSockets();
 940   CancelAllRequestsWithError(error);
 941 }
 942
 943 void ClientSocketPoolBaseHelper::RemoveConnectJob(ConnectJob* job,
 944                                                   Group* group) {
 945   CHECK_GT(connecting_socket_count_, 0);
 946   connecting_socket_count_--;
 947
 948   DCHECK(group);
 949   group->RemoveJob(job);
 950 }
 951
 952 void ClientSocketPoolBaseHelper::OnAvailableSocketSlot(
 953     const std::string& group_name, Group* group) {
 954   DCHECK(ContainsKey(group_map_, group_name));
 955   if (group->IsEmpty()) {
 956     RemoveGroup(group_name);
 957   } else if (group->has_pending_requests()) {
 958     ProcessPendingRequest(group_name, group);
 959   }
 960 }
 961
 962 void ClientSocketPoolBaseHelper::ProcessPendingRequest(
 963     const std::string& group_name, Group* group) {
 964   const Request* next_request = group->GetNextPendingRequest();
 965   DCHECK(next_request);
 966
 967   // If the group has no idle sockets, and can't make use of an additional slot,
 968   // either because it's at the limit or because it's at the socket per group
 969   // limit, then there's nothing to do.
 970   if (group->idle_sockets().empty() &&
 971       !group->CanUseAdditionalSocketSlot(max_sockets_per_group_)) {
 972     return;
 973   }
 974
 975   int rv = RequestSocketInternal(group_name, *next_request);
 976   if (rv != ERR_IO_PENDING) {
 977     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 978     DCHECK(request);
 979     if (group->IsEmpty())
 980       RemoveGroup(group_name);
 981
 982     request->net_log().EndEventWithNetErrorCode(NetLog::TYPE_SOCKET_POOL, rv);
 983     InvokeUserCallbackLater(request->handle(), request->callback(), rv);
 984   }
 985 }
 986
 987 void ClientSocketPoolBaseHelper::HandOutSocket(
 988     scoped_ptr<StreamSocket> socket,
 989     ClientSocketHandle::SocketReuseType reuse_type,
 990     const LoadTimingInfo::ConnectTiming& connect_timing,
 991     ClientSocketHandle* handle,
 992     base::TimeDelta idle_time,
 993     Group* group,
 994     const BoundNetLog& net_log) {
 995   DCHECK(socket);
 996   handle->SetSocket(socket.Pass());
 997   handle->set_reuse_type(reuse_type);
 998   handle->set_idle_time(idle_time);
 999   handle->set_pool_id(pool_generation_number_);
1000   handle->set_connect_timing(connect_timing);
1001
1002   if (handle->is_reused()) {
1003     net_log.AddEvent(
1004         NetLog::TYPE_SOCKET_POOL_REUSED_AN_EXISTING_SOCKET,
1005         NetLog::IntegerCallback(
1006             "idle_ms", static_cast<int>(idle_time.InMilliseconds())));
1007   }
1008
1009   net_log.AddEvent(
1010       NetLog::TYPE_SOCKET_POOL_BOUND_TO_SOCKET,
1011       handle->socket()->NetLog().source().ToEventParametersCallback());
1012
1013   handed_out_socket_count_++;
1014   group->IncrementActiveSocketCount();
1015 }
1016
1017 void ClientSocketPoolBaseHelper::AddIdleSocket(
1018     scoped_ptr<StreamSocket> socket,
1019     Group* group) {
1020   DCHECK(socket);
1021   IdleSocket idle_socket;
1022   idle_socket.socket = socket.release();
1023   idle_socket.start_time = base::TimeTicks::Now();
1024
1025   group->mutable_idle_sockets()->push_back(idle_socket);
1026   IncrementIdleCount();
1027 }
1028
1029 void ClientSocketPoolBaseHelper::CancelAllConnectJobs() {
1030   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end();) {
1031     Group* group = i->second;
1032     connecting_socket_count_ -= group->jobs().size();
1033     group->RemoveAllJobs();
1034
1035     // Delete group if no longer needed.
1036     if (group->IsEmpty()) {
1037       // RemoveGroup() will call .erase() which will invalidate the iterator,
1038       // but i will already have been incremented to a valid iterator before
1039       // RemoveGroup() is called.
1040       RemoveGroup(i++);
1041     } else {
1042       ++i;
1043     }
1044   }
1045   DCHECK_EQ(0, connecting_socket_count_);
1046 }
1047
1048 void ClientSocketPoolBaseHelper::CancelAllRequestsWithError(int error) {
1049   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end();) {
1050     Group* group = i->second;
1051
1052     while (true) {
1053       scoped_ptr<const Request> request = group->PopNextPendingRequest();
1054       if (!request)
1055         break;
1056       InvokeUserCallbackLater(request->handle(), request->callback(), error);
1057     }
1058
1059     // Delete group if no longer needed.
1060     if (group->IsEmpty()) {
1061       // RemoveGroup() will call .erase() which will invalidate the iterator,
1062       // but i will already have been incremented to a valid iterator before
1063       // RemoveGroup() is called.
1064       RemoveGroup(i++);
1065     } else {
1066       ++i;
1067     }
1068   }
1069 }
1070
1071 bool ClientSocketPoolBaseHelper::ReachedMaxSocketsLimit() const {
1072   // Each connecting socket will eventually connect and be handed out.
1073   int total = handed_out_socket_count_ + connecting_socket_count_ +
1074       idle_socket_count();
1075   // There can be more sockets than the limit since some requests can ignore
1076   // the limit
1077   if (total < max_sockets_)
1078     return false;
1079   return true;
1080 }
1081
1082 bool ClientSocketPoolBaseHelper::CloseOneIdleSocket() {
1083   if (idle_socket_count() == 0)
1084     return false;
1085   return CloseOneIdleSocketExceptInGroup(NULL);
1086 }
1087
1088 bool ClientSocketPoolBaseHelper::CloseOneIdleSocketExceptInGroup(
1089     const Group* exception_group) {
1090   CHECK_GT(idle_socket_count(), 0);
1091
1092   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end(); ++i) {
1093     Group* group = i->second;
1094     if (exception_group == group)
1095       continue;
1096     std::list<IdleSocket>* idle_sockets = group->mutable_idle_sockets();
1097
1098     if (!idle_sockets->empty()) {
1099       delete idle_sockets->front().socket;
1100       idle_sockets->pop_front();
1101       DecrementIdleCount();
1102       if (group->IsEmpty())
1103         RemoveGroup(i);
1104
1105       return true;
1106     }
1107   }
1108
1109   return false;
1110 }
1111
1112 bool ClientSocketPoolBaseHelper::CloseOneIdleConnectionInHigherLayeredPool() {
1113   // This pool doesn't have any idle sockets. It's possible that a pool at a
1114   // higher layer is holding one of this sockets active, but it's actually idle.
1115   // Query the higher layers.
1116   for (std::set<HigherLayeredPool*>::const_iterator it = higher_pools_.begin();
1117        it != higher_pools_.end(); ++it) {
1118     if ((*it)->CloseOneIdleConnection())
1119       return true;
1120   }
1121   return false;
1122 }
1123
1124 void ClientSocketPoolBaseHelper::InvokeUserCallbackLater(
1125     ClientSocketHandle* handle, const CompletionCallback& callback, int rv) {
1126   CHECK(!ContainsKey(pending_callback_map_, handle));
1127   pending_callback_map_[handle] = CallbackResultPair(callback, rv);
1128   base::MessageLoop::current()->PostTask(
1129       FROM_HERE,
1130       base::Bind(&ClientSocketPoolBaseHelper::InvokeUserCallback,
1131                  weak_factory_.GetWeakPtr(), handle));
1132 }
1133
1134 void ClientSocketPoolBaseHelper::InvokeUserCallback(
1135     ClientSocketHandle* handle) {
1136   PendingCallbackMap::iterator it = pending_callback_map_.find(handle);
1137
1138   // Exit if the request has already been cancelled.
1139   if (it == pending_callback_map_.end())
1140     return;
1141
1142   CHECK(!handle->is_initialized());
1143   CompletionCallback callback = it->second.callback;
1144   int result = it->second.result;
1145   pending_callback_map_.erase(it);
1146   callback.Run(result);
1147 }
1148
1149 void ClientSocketPoolBaseHelper::TryToCloseSocketsInLayeredPools() {
1150   while (IsStalled()) {
1151     // Closing a socket will result in calling back into |this| to use the freed
1152     // socket slot, so nothing else is needed.
1153     if (!CloseOneIdleConnectionInHigherLayeredPool())
1154       return;
1155   }
1156 }
1157
1158 ClientSocketPoolBaseHelper::Group::Group()
1159     : unassigned_job_count_(0),
1160       pending_requests_(NUM_PRIORITIES),
1161       active_socket_count_(0) {}
1162
1163 ClientSocketPoolBaseHelper::Group::~Group() {
1164   DCHECK_EQ(0u, unassigned_job_count_);
1165 }
1166
1167 void ClientSocketPoolBaseHelper::Group::StartBackupJobTimer(
1168     const std::string& group_name,
1169     ClientSocketPoolBaseHelper* pool) {
1170   // Only allow one timer to run at a time.
1171   if (BackupJobTimerIsRunning())
1172     return;
1173
1174   // Unretained here is okay because |backup_job_timer_| is
1175   // automatically cancelled when it's destroyed.
1176   backup_job_timer_.Start(
1177       FROM_HERE, pool->ConnectRetryInterval(),
1178       base::Bind(&Group::OnBackupJobTimerFired, base::Unretained(this),
1179                  group_name, pool));
1180 }
1181
1182 bool ClientSocketPoolBaseHelper::Group::BackupJobTimerIsRunning() const {
1183   return backup_job_timer_.IsRunning();
1184 }
1185
1186 bool ClientSocketPoolBaseHelper::Group::TryToUseUnassignedConnectJob() {
1187   SanityCheck();
1188
1189   if (unassigned_job_count_ == 0)
1190     return false;
1191   --unassigned_job_count_;
1192   return true;
1193 }
1194
1195 void ClientSocketPoolBaseHelper::Group::AddJob(scoped_ptr<ConnectJob> job,
1196                                                bool is_preconnect) {
1197   SanityCheck();
1198
1199   if (is_preconnect)
1200     ++unassigned_job_count_;
1201   jobs_.insert(job.release());
1202 }
1203
1204 void ClientSocketPoolBaseHelper::Group::RemoveJob(ConnectJob* job) {
1205   scoped_ptr<ConnectJob> owned_job(job);
1206   SanityCheck();
1207
1208   std::set<ConnectJob*>::iterator it = jobs_.find(job);
1209   if (it != jobs_.end()) {
1210     jobs_.erase(it);
1211   } else {
1212     NOTREACHED();
1213   }
1214   size_t job_count = jobs_.size();
1215   if (job_count < unassigned_job_count_)
1216     unassigned_job_count_ = job_count;
1217
1218   // If we've got no more jobs for this group, then we no longer need a
1219   // backup job either.
1220   if (jobs_.empty())
1221     backup_job_timer_.Stop();
1222 }
1223
1224 void ClientSocketPoolBaseHelper::Group::OnBackupJobTimerFired(
1225     std::string group_name,
1226     ClientSocketPoolBaseHelper* pool) {
1227   // If there are no more jobs pending, there is no work to do.
1228   // If we've done our cleanups correctly, this should not happen.
1229   if (jobs_.empty()) {
1230     NOTREACHED();
1231     return;
1232   }
1233
1234   // If our old job is waiting on DNS, or if we can't create any sockets
1235   // right now due to limits, just reset the timer.
1236   if (pool->ReachedMaxSocketsLimit() ||
1237       !HasAvailableSocketSlot(pool->max_sockets_per_group_) ||
1238       (*jobs_.begin())->GetLoadState() == LOAD_STATE_RESOLVING_HOST) {
1239     StartBackupJobTimer(group_name, pool);
1240     return;
1241   }
1242
1243   if (pending_requests_.empty())
1244     return;
1245
1246   scoped_ptr<ConnectJob> backup_job =
1247       pool->connect_job_factory_->NewConnectJob(
1248           group_name, *pending_requests_.FirstMax().value(), pool);
1249   backup_job->net_log().AddEvent(NetLog::TYPE_BACKUP_CONNECT_JOB_CREATED);
1250   int rv = backup_job->Connect();
1251   pool->connecting_socket_count_++;
1252   ConnectJob* raw_backup_job = backup_job.get();
1253   AddJob(backup_job.Pass(), false);
1254   if (rv != ERR_IO_PENDING)
1255     pool->OnConnectJobComplete(rv, raw_backup_job);
1256 }
1257
1258 void ClientSocketPoolBaseHelper::Group::SanityCheck() {
1259   DCHECK_LE(unassigned_job_count_, jobs_.size());
1260 }
1261
1262 void ClientSocketPoolBaseHelper::Group::RemoveAllJobs() {
1263   SanityCheck();
1264
1265   // Delete active jobs.
1266   STLDeleteElements(&jobs_);
1267   unassigned_job_count_ = 0;
1268
1269   // Stop backup job timer.
1270   backup_job_timer_.Stop();
1271 }
1272
1273 const ClientSocketPoolBaseHelper::Request*
1274 ClientSocketPoolBaseHelper::Group::GetNextPendingRequest() const {
1275   return
1276       pending_requests_.empty() ? NULL : pending_requests_.FirstMax().value();
1277 }
1278
1279 bool ClientSocketPoolBaseHelper::Group::HasConnectJobForHandle(
1280     const ClientSocketHandle* handle) const {
1281   // Search the first |jobs_.size()| pending requests for |handle|.
1282   // If it's farther back in the deque than that, it doesn't have a
1283   // corresponding ConnectJob.
1284   size_t i = 0;
1285   for (RequestQueue::Pointer pointer = pending_requests_.FirstMax();
1286        !pointer.is_null() && i < jobs_.size();
1287        pointer = pending_requests_.GetNextTowardsLastMin(pointer), ++i) {
1288     if (pointer.value()->handle() == handle)
1289       return true;
1290   }
1291   return false;
1292 }
1293
1294 void ClientSocketPoolBaseHelper::Group::InsertPendingRequest(
1295     scoped_ptr<const Request> request) {
1296   // This value must be cached before we release |request|.
1297   RequestPriority priority = request->priority();
1298   if (request->ignore_limits()) {
1299     // Put requests with ignore_limits == true (which should have
1300     // priority == MAXIMUM_PRIORITY) ahead of other requests with
1301     // MAXIMUM_PRIORITY.
1302     DCHECK_EQ(priority, MAXIMUM_PRIORITY);
1303     pending_requests_.InsertAtFront(request.release(), priority);
1304   } else {
1305     pending_requests_.Insert(request.release(), priority);
1306   }
1307 }
1308
1309 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1310 ClientSocketPoolBaseHelper::Group::PopNextPendingRequest() {
1311   if (pending_requests_.empty())
1312     return scoped_ptr<const ClientSocketPoolBaseHelper::Request>();
1313   return RemovePendingRequest(pending_requests_.FirstMax());
1314 }
1315
1316 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1317 ClientSocketPoolBaseHelper::Group::FindAndRemovePendingRequest(
1318     ClientSocketHandle* handle) {
1319   for (RequestQueue::Pointer pointer = pending_requests_.FirstMax();
1320        !pointer.is_null();
1321        pointer = pending_requests_.GetNextTowardsLastMin(pointer)) {
1322     if (pointer.value()->handle() == handle) {
1323       scoped_ptr<const Request> request = RemovePendingRequest(pointer);
1324       return request.Pass();
1325     }
1326   }
1327   return scoped_ptr<const ClientSocketPoolBaseHelper::Request>();
1328 }
1329
1330 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1331 ClientSocketPoolBaseHelper::Group::RemovePendingRequest(
1332     const RequestQueue::Pointer& pointer) {
1333   scoped_ptr<const Request> request(pointer.value());
1334   pending_requests_.Erase(pointer);
1335   // If there are no more requests, kill the backup timer.
1336   if (pending_requests_.empty())
1337     backup_job_timer_.Stop();
1338   return request.Pass();
1339 }
1340
1341 }  // namespace internal
1342
1343 }  // namespace net