1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "google_apis/gcm/engine/connection_factory_impl.h"
7 #include "base/message_loop/message_loop.h"
8 #include "base/metrics/histogram.h"
9 #include "base/metrics/sparse_histogram.h"
10 #include "google_apis/gcm/engine/connection_handler_impl.h"
11 #include "google_apis/gcm/monitoring/gcm_stats_recorder.h"
12 #include "google_apis/gcm/protocol/mcs.pb.h"
13 #include "net/base/load_flags.h"
14 #include "net/base/net_errors.h"
15 #include "net/http/http_network_session.h"
16 #include "net/http/http_request_headers.h"
17 #include "net/proxy/proxy_info.h"
18 #include "net/socket/client_socket_handle.h"
19 #include "net/socket/client_socket_pool_manager.h"
20 #include "net/ssl/ssl_config_service.h"
26 // The amount of time a Socket read should wait before timing out.
27 const int kReadTimeoutMs
= 30000; // 30 seconds.
29 // If a connection is reset after succeeding within this window of time,
30 // the previous backoff entry is restored (and the connection success is treated
31 // as if it was transient).
32 const int kConnectionResetWindowSecs
= 10; // 10 seconds.
34 // Decides whether the last login was within kConnectionResetWindowSecs of now
36 bool ShouldRestorePreviousBackoff(const base::TimeTicks
& login_time
,
37 const base::TimeTicks
& now_ticks
) {
38 return !login_time
.is_null() &&
39 now_ticks
- login_time
<=
40 base::TimeDelta::FromSeconds(kConnectionResetWindowSecs
);
45 ConnectionFactoryImpl::ConnectionFactoryImpl(
46 const std::vector
<GURL
>& mcs_endpoints
,
47 const net::BackoffEntry::Policy
& backoff_policy
,
48 const scoped_refptr
<net::HttpNetworkSession
>& gcm_network_session
,
49 const scoped_refptr
<net::HttpNetworkSession
>& http_network_session
,
51 GCMStatsRecorder
* recorder
)
52 : mcs_endpoints_(mcs_endpoints
),
54 last_successful_endpoint_(0),
55 backoff_policy_(backoff_policy
),
56 gcm_network_session_(gcm_network_session
),
57 http_network_session_(http_network_session
),
59 net::BoundNetLog::Make(net_log
, net::NetLog::SOURCE_SOCKET
)),
62 waiting_for_backoff_(false),
63 waiting_for_network_online_(false),
67 weak_ptr_factory_(this) {
68 DCHECK_GE(mcs_endpoints_
.size(), 1U);
69 DCHECK(!http_network_session_
.get() ||
70 (gcm_network_session_
.get() != http_network_session_
.get()));
73 ConnectionFactoryImpl::~ConnectionFactoryImpl() {
75 net::NetworkChangeNotifier::RemoveNetworkChangeObserver(this);
77 gcm_network_session_
->proxy_service()->CancelPacRequest(pac_request_
);
82 void ConnectionFactoryImpl::Initialize(
83 const BuildLoginRequestCallback
& request_builder
,
84 const ConnectionHandler::ProtoReceivedCallback
& read_callback
,
85 const ConnectionHandler::ProtoSentCallback
& write_callback
) {
86 DCHECK(!connection_handler_
);
87 DCHECK(read_callback_
.is_null());
88 DCHECK(write_callback_
.is_null());
90 previous_backoff_
= CreateBackoffEntry(&backoff_policy_
);
91 backoff_entry_
= CreateBackoffEntry(&backoff_policy_
);
92 request_builder_
= request_builder
;
93 read_callback_
= read_callback
;
94 write_callback_
= write_callback
;
96 net::NetworkChangeNotifier::AddNetworkChangeObserver(this);
97 waiting_for_network_online_
= net::NetworkChangeNotifier::IsOffline();
100 ConnectionHandler
* ConnectionFactoryImpl::GetConnectionHandler() const {
101 return connection_handler_
.get();
104 void ConnectionFactoryImpl::Connect() {
105 if (!connection_handler_
) {
106 connection_handler_
= CreateConnectionHandler(
107 base::TimeDelta::FromMilliseconds(kReadTimeoutMs
),
110 base::Bind(&ConnectionFactoryImpl::ConnectionHandlerCallback
,
111 weak_ptr_factory_
.GetWeakPtr())).Pass();
114 if (connecting_
|| waiting_for_backoff_
)
115 return; // Connection attempt already in progress or pending.
117 if (IsEndpointReachable())
118 return; // Already connected.
120 ConnectWithBackoff();
123 void ConnectionFactoryImpl::ConnectWithBackoff() {
124 // If a canary managed to connect while a backoff expiration was pending,
125 // just cleanup the internal state.
126 if (connecting_
|| logging_in_
|| IsEndpointReachable()) {
127 waiting_for_backoff_
= false;
131 if (backoff_entry_
->ShouldRejectRequest()) {
132 DVLOG(1) << "Delaying MCS endpoint connection for "
133 << backoff_entry_
->GetTimeUntilRelease().InMilliseconds()
135 waiting_for_backoff_
= true;
136 recorder_
->RecordConnectionDelayedDueToBackoff(
137 backoff_entry_
->GetTimeUntilRelease().InMilliseconds());
138 base::MessageLoop::current()->PostDelayedTask(
140 base::Bind(&ConnectionFactoryImpl::ConnectWithBackoff
,
141 weak_ptr_factory_
.GetWeakPtr()),
142 backoff_entry_
->GetTimeUntilRelease());
146 DVLOG(1) << "Attempting connection to MCS endpoint.";
147 waiting_for_backoff_
= false;
151 bool ConnectionFactoryImpl::IsEndpointReachable() const {
152 return connection_handler_
&& connection_handler_
->CanSendMessage();
155 std::string
ConnectionFactoryImpl::GetConnectionStateString() const {
156 if (IsEndpointReachable())
162 if (waiting_for_backoff_
)
163 return "WAITING FOR BACKOFF";
164 if (waiting_for_network_online_
)
165 return "WAITING FOR NETWORK CHANGE";
166 return "NOT CONNECTED";
169 void ConnectionFactoryImpl::SignalConnectionReset(
170 ConnectionResetReason reason
) {
171 if (!connection_handler_
) {
172 // No initial connection has been made. No need to do anything.
176 // A failure can trigger multiple resets, so no need to do anything if a
177 // connection is already in progress.
179 DVLOG(1) << "Connection in progress, ignoring reset.";
184 listener_
->OnDisconnected();
186 UMA_HISTOGRAM_ENUMERATION("GCM.ConnectionResetReason",
188 CONNECTION_RESET_COUNT
);
189 recorder_
->RecordConnectionResetSignaled(reason
);
190 if (!last_login_time_
.is_null()) {
191 UMA_HISTOGRAM_CUSTOM_TIMES("GCM.ConnectionUpTime",
192 NowTicks() - last_login_time_
,
193 base::TimeDelta::FromSeconds(1),
194 base::TimeDelta::FromHours(24),
196 // |last_login_time_| will be reset below, before attempting the new
201 DCHECK(!IsEndpointReachable());
203 // TODO(zea): if the network is offline, don't attempt to connect.
204 // See crbug.com/396687
206 // Network changes get special treatment as they can trigger a one-off canary
207 // request that bypasses backoff (but does nothing if a connection is in
208 // progress). Other connection reset events can be ignored as a connection
209 // is already awaiting backoff expiration.
210 if (waiting_for_backoff_
&& reason
!= NETWORK_CHANGE
) {
211 DVLOG(1) << "Backoff expiration pending, ignoring reset.";
216 // Failures prior to login completion just reuse the existing backoff entry.
218 backoff_entry_
->InformOfRequest(false);
219 } else if (reason
== LOGIN_FAILURE
||
220 ShouldRestorePreviousBackoff(last_login_time_
, NowTicks())) {
221 // Failures due to login, or within the reset window of a login, restore
222 // the backoff entry that was saved off at login completion time.
223 backoff_entry_
.swap(previous_backoff_
);
224 backoff_entry_
->InformOfRequest(false);
225 } else if (reason
== NETWORK_CHANGE
) {
226 ConnectImpl(); // Canary attempts bypass backoff without resetting it.
229 // We shouldn't be in backoff in thise case.
230 DCHECK_EQ(0, backoff_entry_
->failure_count());
233 // At this point the last login time has been consumed or deemed irrelevant,
235 last_login_time_
= base::TimeTicks();
240 void ConnectionFactoryImpl::SetConnectionListener(
241 ConnectionListener
* listener
) {
242 listener_
= listener
;
245 base::TimeTicks
ConnectionFactoryImpl::NextRetryAttempt() const {
247 return base::TimeTicks();
248 return backoff_entry_
->GetReleaseTime();
251 void ConnectionFactoryImpl::OnNetworkChanged(
252 net::NetworkChangeNotifier::ConnectionType type
) {
253 if (type
== net::NetworkChangeNotifier::CONNECTION_NONE
) {
254 DVLOG(1) << "Network lost, resettion connection.";
255 waiting_for_network_online_
= true;
257 // Will do nothing due to |waiting_for_network_online_ == true|.
258 // TODO(zea): make the above statement actually true. See crbug.com/396687
259 SignalConnectionReset(NETWORK_CHANGE
);
263 DVLOG(1) << "Connection type changed to " << type
<< ", reconnecting.";
264 waiting_for_network_online_
= false;
265 SignalConnectionReset(NETWORK_CHANGE
);
268 GURL
ConnectionFactoryImpl::GetCurrentEndpoint() const {
269 // Note that IsEndpointReachable() returns false anytime connecting_ is true,
270 // so while connecting this always uses |next_endpoint_|.
271 if (IsEndpointReachable())
272 return mcs_endpoints_
[last_successful_endpoint_
];
273 return mcs_endpoints_
[next_endpoint_
];
276 net::IPEndPoint
ConnectionFactoryImpl::GetPeerIP() {
277 if (!socket_handle_
.socket())
278 return net::IPEndPoint();
280 net::IPEndPoint ip_endpoint
;
281 int result
= socket_handle_
.socket()->GetPeerAddress(&ip_endpoint
);
282 if (result
!= net::OK
)
283 return net::IPEndPoint();
288 void ConnectionFactoryImpl::ConnectImpl() {
289 DCHECK(!IsEndpointReachable());
290 DCHECK(!socket_handle_
.socket());
292 // TODO(zea): if the network is offline, don't attempt to connect.
293 // See crbug.com/396687
296 GURL current_endpoint
= GetCurrentEndpoint();
297 recorder_
->RecordConnectionInitiated(current_endpoint
.host());
298 RebuildNetworkSessionAuthCache();
299 int status
= gcm_network_session_
->proxy_service()->ResolveProxy(
303 base::Bind(&ConnectionFactoryImpl::OnProxyResolveDone
,
304 weak_ptr_factory_
.GetWeakPtr()),
308 if (status
!= net::ERR_IO_PENDING
)
309 OnProxyResolveDone(status
);
312 void ConnectionFactoryImpl::InitHandler() {
313 // May be null in tests.
314 mcs_proto::LoginRequest login_request
;
315 if (!request_builder_
.is_null()) {
316 request_builder_
.Run(&login_request
);
317 DCHECK(login_request
.IsInitialized());
320 connection_handler_
->Init(login_request
, socket_handle_
.socket());
323 scoped_ptr
<net::BackoffEntry
> ConnectionFactoryImpl::CreateBackoffEntry(
324 const net::BackoffEntry::Policy
* const policy
) {
325 return scoped_ptr
<net::BackoffEntry
>(new net::BackoffEntry(policy
));
328 scoped_ptr
<ConnectionHandler
> ConnectionFactoryImpl::CreateConnectionHandler(
329 base::TimeDelta read_timeout
,
330 const ConnectionHandler::ProtoReceivedCallback
& read_callback
,
331 const ConnectionHandler::ProtoSentCallback
& write_callback
,
332 const ConnectionHandler::ConnectionChangedCallback
& connection_callback
) {
333 return make_scoped_ptr
<ConnectionHandler
>(
334 new ConnectionHandlerImpl(read_timeout
,
337 connection_callback
));
340 base::TimeTicks
ConnectionFactoryImpl::NowTicks() {
341 return base::TimeTicks::Now();
344 void ConnectionFactoryImpl::OnConnectDone(int result
) {
345 if (result
!= net::OK
) {
346 // If the connection fails, try another proxy.
347 result
= ReconsiderProxyAfterError(result
);
348 // ReconsiderProxyAfterError either returns an error (in which case it is
349 // not reconsidering a proxy) or returns ERR_IO_PENDING if it is considering
351 DCHECK_NE(result
, net::OK
);
352 if (result
== net::ERR_IO_PENDING
)
353 return; // Proxy reconsideration pending. Return.
354 LOG(ERROR
) << "Failed to connect to MCS endpoint with error " << result
;
355 UMA_HISTOGRAM_BOOLEAN("GCM.ConnectionSuccessRate", false);
356 recorder_
->RecordConnectionFailure(result
);
358 backoff_entry_
->InformOfRequest(false);
359 UMA_HISTOGRAM_SPARSE_SLOWLY("GCM.ConnectionFailureErrorCode", result
);
361 // If there are other endpoints available, use the next endpoint on the
364 if (next_endpoint_
>= mcs_endpoints_
.size())
371 UMA_HISTOGRAM_BOOLEAN("GCM.ConnectionSuccessRate", true);
372 UMA_HISTOGRAM_COUNTS("GCM.ConnectionEndpoint", next_endpoint_
);
373 UMA_HISTOGRAM_BOOLEAN("GCM.ConnectedViaProxy",
374 !(proxy_info_
.is_empty() || proxy_info_
.is_direct()));
375 ReportSuccessfulProxyConnection();
376 recorder_
->RecordConnectionSuccess();
378 // Reset the endpoint back to the default.
379 // TODO(zea): consider prioritizing endpoints more intelligently based on
380 // which ones succeed most for this client? Although that will affect
381 // measuring the success rate of the default endpoint vs fallback.
382 last_successful_endpoint_
= next_endpoint_
;
386 DVLOG(1) << "MCS endpoint socket connection success, starting login.";
390 void ConnectionFactoryImpl::ConnectionHandlerCallback(int result
) {
391 DCHECK(!connecting_
);
392 if (result
!= net::OK
) {
393 // TODO(zea): Consider how to handle errors that may require some sort of
394 // user intervention (login page, etc.).
395 UMA_HISTOGRAM_SPARSE_SLOWLY("GCM.ConnectionDisconnectErrorCode", result
);
396 SignalConnectionReset(SOCKET_FAILURE
);
400 // Handshake complete, reset backoff. If the login failed with an error,
401 // the client should invoke SignalConnectionReset(LOGIN_FAILURE), which will
402 // restore the previous backoff.
403 DVLOG(1) << "Handshake complete.";
404 last_login_time_
= NowTicks();
405 previous_backoff_
.swap(backoff_entry_
);
406 backoff_entry_
->Reset();
410 listener_
->OnConnected(GetCurrentEndpoint(), GetPeerIP());
413 // This has largely been copied from
414 // HttpStreamFactoryImpl::Job::DoResolveProxyComplete. This should be
415 // refactored into some common place.
416 void ConnectionFactoryImpl::OnProxyResolveDone(int status
) {
418 DVLOG(1) << "Proxy resolution status: " << status
;
420 DCHECK_NE(status
, net::ERR_IO_PENDING
);
421 if (status
== net::OK
) {
422 // Remove unsupported proxies from the list.
423 proxy_info_
.RemoveProxiesWithoutScheme(
424 net::ProxyServer::SCHEME_DIRECT
|
425 net::ProxyServer::SCHEME_HTTP
| net::ProxyServer::SCHEME_HTTPS
|
426 net::ProxyServer::SCHEME_SOCKS4
| net::ProxyServer::SCHEME_SOCKS5
);
428 if (proxy_info_
.is_empty()) {
429 // No proxies/direct to choose from. This happens when we don't support
430 // any of the proxies in the returned list.
431 status
= net::ERR_NO_SUPPORTED_PROXIES
;
435 if (status
!= net::OK
) {
436 // Failed to resolve proxy. Retry later.
437 OnConnectDone(status
);
441 DVLOG(1) << "Resolved proxy with PAC:" << proxy_info_
.ToPacString();
443 net::SSLConfig ssl_config
;
444 gcm_network_session_
->ssl_config_service()->GetSSLConfig(&ssl_config
);
445 status
= net::InitSocketHandleForTlsConnect(
446 net::HostPortPair::FromURL(GetCurrentEndpoint()),
447 gcm_network_session_
.get(),
451 net::PRIVACY_MODE_DISABLED
,
454 base::Bind(&ConnectionFactoryImpl::OnConnectDone
,
455 weak_ptr_factory_
.GetWeakPtr()));
456 if (status
!= net::ERR_IO_PENDING
)
457 OnConnectDone(status
);
460 // This has largely been copied from
461 // HttpStreamFactoryImpl::Job::ReconsiderProxyAfterError. This should be
462 // refactored into some common place.
463 // This method reconsiders the proxy on certain errors. If it does reconsider
464 // a proxy it always returns ERR_IO_PENDING and posts a call to
465 // OnProxyResolveDone with the result of the reconsideration.
466 int ConnectionFactoryImpl::ReconsiderProxyAfterError(int error
) {
467 DCHECK(!pac_request_
);
468 DCHECK_NE(error
, net::OK
);
469 DCHECK_NE(error
, net::ERR_IO_PENDING
);
470 // A failure to resolve the hostname or any error related to establishing a
471 // TCP connection could be grounds for trying a new proxy configuration.
473 // Why do this when a hostname cannot be resolved? Some URLs only make sense
474 // to proxy servers. The hostname in those URLs might fail to resolve if we
475 // are still using a non-proxy config. We need to check if a proxy config
476 // now exists that corresponds to a proxy server that could load the URL.
479 case net::ERR_PROXY_CONNECTION_FAILED
:
480 case net::ERR_NAME_NOT_RESOLVED
:
481 case net::ERR_INTERNET_DISCONNECTED
:
482 case net::ERR_ADDRESS_UNREACHABLE
:
483 case net::ERR_CONNECTION_CLOSED
:
484 case net::ERR_CONNECTION_TIMED_OUT
:
485 case net::ERR_CONNECTION_RESET
:
486 case net::ERR_CONNECTION_REFUSED
:
487 case net::ERR_CONNECTION_ABORTED
:
488 case net::ERR_TIMED_OUT
:
489 case net::ERR_TUNNEL_CONNECTION_FAILED
:
490 case net::ERR_SOCKS_CONNECTION_FAILED
:
491 // This can happen in the case of trying to talk to a proxy using SSL, and
492 // ending up talking to a captive portal that supports SSL instead.
493 case net::ERR_PROXY_CERTIFICATE_INVALID
:
494 // This can happen when trying to talk SSL to a non-SSL server (Like a
496 case net::ERR_SSL_PROTOCOL_ERROR
:
498 case net::ERR_SOCKS_CONNECTION_HOST_UNREACHABLE
:
499 // Remap the SOCKS-specific "host unreachable" error to a more
500 // generic error code (this way consumers like the link doctor
501 // know to substitute their error page).
503 // Note that if the host resolving was done by the SOCKS5 proxy, we can't
504 // differentiate between a proxy-side "host not found" versus a proxy-side
505 // "address unreachable" error, and will report both of these failures as
506 // ERR_ADDRESS_UNREACHABLE.
507 return net::ERR_ADDRESS_UNREACHABLE
;
512 net::SSLConfig ssl_config
;
513 gcm_network_session_
->ssl_config_service()->GetSSLConfig(&ssl_config
);
514 if (proxy_info_
.is_https() && ssl_config
.send_client_cert
) {
515 gcm_network_session_
->ssl_client_auth_cache()->Remove(
516 proxy_info_
.proxy_server().host_port_pair());
519 int status
= gcm_network_session_
->proxy_service()->ReconsiderProxyAfterError(
520 GetCurrentEndpoint(), net::LOAD_NORMAL
, error
, &proxy_info_
,
521 base::Bind(&ConnectionFactoryImpl::OnProxyResolveDone
,
522 weak_ptr_factory_
.GetWeakPtr()),
526 if (status
== net::OK
|| status
== net::ERR_IO_PENDING
) {
529 // If ReconsiderProxyAfterError() failed synchronously, it means
530 // there was nothing left to fall-back to, so fail the transaction
531 // with the last connection error we got.
535 // If there is new proxy info, post OnProxyResolveDone to retry it. Otherwise,
536 // if there was an error falling back, fail synchronously.
537 if (status
== net::OK
) {
538 base::MessageLoop::current()->PostTask(
540 base::Bind(&ConnectionFactoryImpl::OnProxyResolveDone
,
541 weak_ptr_factory_
.GetWeakPtr(), status
));
542 status
= net::ERR_IO_PENDING
;
547 void ConnectionFactoryImpl::ReportSuccessfulProxyConnection() {
548 if (gcm_network_session_
.get() && gcm_network_session_
->proxy_service())
549 gcm_network_session_
->proxy_service()->ReportSuccess(proxy_info_
, NULL
);
552 void ConnectionFactoryImpl::CloseSocket() {
553 // The connection handler needs to be reset, else it'll attempt to keep using
554 // the destroyed socket.
555 if (connection_handler_
)
556 connection_handler_
->Reset();
558 if (socket_handle_
.socket() && socket_handle_
.socket()->IsConnected())
559 socket_handle_
.socket()->Disconnect();
560 socket_handle_
.Reset();
563 void ConnectionFactoryImpl::RebuildNetworkSessionAuthCache() {
564 if (!http_network_session_
.get() || !http_network_session_
->http_auth_cache())
567 gcm_network_session_
->http_auth_cache()->UpdateAllFrom(
568 *http_network_session_
->http_auth_cache());