1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef CHROME_BROWSER_SAFE_BROWSING_PROTOCOL_MANAGER_H_
6 #define CHROME_BROWSER_SAFE_BROWSING_PROTOCOL_MANAGER_H_
8 // A class that implements Chrome's interface with the SafeBrowsing protocol.
9 // See https://developers.google.com/safe-browsing/developers_guide_v2 for
12 // The SafeBrowsingProtocolManager handles formatting and making requests of,
13 // and handling responses from, Google's SafeBrowsing servers. This class uses
14 // The SafeBrowsingProtocolParser class to do the actual parsing.
21 #include "base/containers/hash_tables.h"
22 #include "base/gtest_prod_util.h"
23 #include "base/memory/scoped_ptr.h"
24 #include "base/threading/non_thread_safe.h"
25 #include "base/time/time.h"
26 #include "base/timer/timer.h"
27 #include "chrome/browser/safe_browsing/chunk_range.h"
28 #include "chrome/browser/safe_browsing/protocol_manager_helper.h"
29 #include "chrome/browser/safe_browsing/protocol_parser.h"
30 #include "chrome/browser/safe_browsing/safe_browsing_util.h"
31 #include "net/url_request/url_fetcher_delegate.h"
32 #include "net/url_request/url_request_status.h"
37 class URLRequestContextGetter
;
40 class SBProtocolManagerFactory
;
41 class SafeBrowsingProtocolManagerDelegate
;
43 class SafeBrowsingProtocolManager
: public net::URLFetcherDelegate
,
44 public base::NonThreadSafe
{
46 // FullHashCallback is invoked when GetFullHash completes.
48 // - The vector of full hash results. If empty, indicates that there
49 // were no matches, and that the resource is safe.
50 // - The cache lifetime of the result. A lifetime of 0 indicates the results
51 // should not be cached.
52 typedef base::Callback
<void(const std::vector
<SBFullHashResult
>&,
53 const base::TimeDelta
&)> FullHashCallback
;
55 ~SafeBrowsingProtocolManager() override
;
57 // Makes the passed |factory| the factory used to instantiate
58 // a SafeBrowsingService. Useful for tests.
59 static void RegisterFactory(SBProtocolManagerFactory
* factory
) {
63 // Create an instance of the safe browsing protocol manager.
64 static SafeBrowsingProtocolManager
* Create(
65 SafeBrowsingProtocolManagerDelegate
* delegate
,
66 net::URLRequestContextGetter
* request_context_getter
,
67 const SafeBrowsingProtocolConfig
& config
);
69 // Sets up the update schedule and internal state for making periodic requests
70 // of the Safebrowsing servers.
71 virtual void Initialize();
73 // net::URLFetcherDelegate interface.
74 void OnURLFetchComplete(const net::URLFetcher
* source
) override
;
76 // Retrieve the full hash for a set of prefixes, and invoke the callback
77 // argument when the results are retrieved. The callback may be invoked
79 virtual void GetFullHash(const std::vector
<SBPrefix
>& prefixes
,
80 FullHashCallback callback
,
82 bool is_extended_reporting
);
84 // Forces the start of next update after |interval| time.
85 void ForceScheduleNextUpdate(base::TimeDelta interval
);
87 // Scheduled update callback.
90 // Called by the SafeBrowsingService when our request for a list of all chunks
91 // for each list is done. If database_error is true, that means the protocol
92 // manager shouldn't fetch updates since they can't be written to disk. It
93 // should try again later to open the database.
94 void OnGetChunksComplete(const std::vector
<SBListChunkRanges
>& list
,
96 bool is_extended_reporting
);
98 // The last time we received an update.
99 base::Time
last_update() const { return last_update_
; }
101 // Setter for additional_query_. To make sure the additional_query_ won't
102 // be changed in the middle of an update, caller (e.g.: SafeBrowsingService)
103 // should call this after callbacks triggered in UpdateFinished() or before
104 // IssueUpdateRequest().
105 void set_additional_query(const std::string
& query
) {
106 additional_query_
= query
;
108 const std::string
& additional_query() const {
109 return additional_query_
;
112 // Enumerate failures for histogramming purposes. DO NOT CHANGE THE
113 // ORDERING OF THESE VALUES.
115 // 200 response code means that the server recognized the hash
116 // prefix, while 204 is an empty response indicating that the
117 // server did not recognize it.
121 // Subset of successful responses which returned no full hashes.
122 // This includes the STATUS_204 case, and the *_ERROR cases.
123 GET_HASH_FULL_HASH_EMPTY
,
125 // Subset of successful responses for which one or more of the
126 // full hashes matched (should lead to an interstitial).
127 GET_HASH_FULL_HASH_HIT
,
129 // Subset of successful responses which weren't empty and have no
130 // matches. It means that there was a prefix collision which was
131 // cleared up by the full hashes.
132 GET_HASH_FULL_HASH_MISS
,
134 // Subset of successful responses where the response body wasn't parsable.
135 GET_HASH_PARSE_ERROR
,
137 // Gethash request failed (network error).
138 GET_HASH_NETWORK_ERROR
,
140 // Gethash request returned HTTP result code other than 200 or 204.
143 // Gethash attempted during error backoff, no request sent.
144 GET_HASH_BACKOFF_ERROR
,
146 // Memory space for histograms is determined by the max. ALWAYS
147 // ADD NEW VALUES BEFORE THIS ONE.
151 // Record a GetHash result. |is_download| indicates if the get
152 // hash is triggered by download related lookup.
153 static void RecordGetHashResult(bool is_download
,
154 ResultType result_type
);
156 // Record HTTP response code when there's no error in fetching hash results,
157 // and the error code, when there is.
158 // |status| represents the status of the HTTP request, and |response code|
159 // represents the HTTP response code received from the server.
160 static void RecordGetHashResponseOrErrorCode(net::URLRequestStatus status
,
163 // Returns whether another update is currently scheduled.
164 bool IsUpdateScheduled() const;
166 // Called when app changes status of foreground or background.
167 void SetAppInForeground(bool foreground
) {
168 app_in_foreground_
= foreground
;
172 // Constructs a SafeBrowsingProtocolManager for |delegate| that issues
173 // network requests using |request_context_getter|.
174 SafeBrowsingProtocolManager(
175 SafeBrowsingProtocolManagerDelegate
* delegate
,
176 net::URLRequestContextGetter
* request_context_getter
,
177 const SafeBrowsingProtocolConfig
& config
);
180 FRIEND_TEST_ALL_PREFIXES(SafeBrowsingProtocolManagerTest
, TestBackOffTimes
);
181 FRIEND_TEST_ALL_PREFIXES(SafeBrowsingProtocolManagerTest
, TestChunkStrings
);
182 FRIEND_TEST_ALL_PREFIXES(SafeBrowsingProtocolManagerTest
, TestGetHashUrl
);
183 FRIEND_TEST_ALL_PREFIXES(SafeBrowsingProtocolManagerTest
,
184 TestGetHashBackOffTimes
);
185 FRIEND_TEST_ALL_PREFIXES(SafeBrowsingProtocolManagerTest
, TestNextChunkUrl
);
186 FRIEND_TEST_ALL_PREFIXES(SafeBrowsingProtocolManagerTest
, TestUpdateUrl
);
187 friend class SafeBrowsingServerTest
;
188 friend class SBProtocolManagerFactoryImpl
;
190 // Internal API for fetching information from the SafeBrowsing servers. The
191 // GetHash requests are higher priority since they can block user requests
192 // so are handled separately.
193 enum SafeBrowsingRequestType
{
194 NO_REQUEST
= 0, // No requests in progress
195 UPDATE_REQUEST
, // Request for redirect URLs
196 BACKUP_UPDATE_REQUEST
, // Request for redirect URLs to a backup URL.
197 CHUNK_REQUEST
, // Request for a specific chunk
200 // Which type of backup update request is being used.
201 enum BackupUpdateReason
{
202 BACKUP_UPDATE_REASON_CONNECT
,
203 BACKUP_UPDATE_REASON_HTTP
,
204 BACKUP_UPDATE_REASON_NETWORK
,
205 BACKUP_UPDATE_REASON_MAX
,
208 // Generates Update URL for querying about the latest set of chunk updates.
209 GURL
UpdateUrl(bool is_extended_reporting
) const;
211 // Generates backup Update URL for querying about the latest set of chunk
212 // updates. |url_prefix| is the base prefix to use.
213 GURL
BackupUpdateUrl(BackupUpdateReason reason
) const;
215 // Generates GetHash request URL for retrieving full hashes.
216 GURL
GetHashUrl(bool is_extended_reporting
) const;
217 // Generates URL for reporting safe browsing hits for UMA users.
219 // Composes a ChunkUrl based on input string.
220 GURL
NextChunkUrl(const std::string
& input
) const;
222 // Returns the time for the next update request. If |back_off| is true,
223 // the time returned will increment an error count and return the appriate
224 // next time (see ScheduleNextUpdate below).
225 base::TimeDelta
GetNextUpdateInterval(bool back_off
);
227 // Worker function for calculating GetHash and Update backoff times (in
228 // seconds). |multiplier| is doubled for each consecutive error between the
229 // 2nd and 5th, and |error_count| is incremented with each call.
230 base::TimeDelta
GetNextBackOffInterval(size_t* error_count
,
231 size_t* multiplier
) const;
233 // Manages our update with the next allowable update time. If 'back_off_' is
234 // true, we must decrease the frequency of requests of the SafeBrowsing
235 // service according to section 5 of the protocol specification.
236 // When disable_auto_update_ is set, ScheduleNextUpdate will do nothing.
237 // ForceScheduleNextUpdate has to be called to trigger the update.
238 void ScheduleNextUpdate(bool back_off
);
240 // Sends a request for a list of chunks we should download to the SafeBrowsing
241 // servers. In order to format this request, we need to send all the chunk
242 // numbers for each list that we have to the server. Getting the chunk numbers
243 // requires a database query (run on the database thread), and the request
244 // is sent upon completion of that query in OnGetChunksComplete.
245 void IssueUpdateRequest();
247 // Sends a backup request for a list of chunks to download, when the primary
248 // update request failed. |reason| specifies why the backup is needed. Unlike
249 // the primary IssueUpdateRequest, this does not need to hit the local
250 // SafeBrowsing database since the existing chunk numbers are remembered from
251 // the primary update request. Returns whether the backup request was issued -
252 // this may be false in cases where there is not a prefix specified.
253 bool IssueBackupUpdateRequest(BackupUpdateReason reason
);
255 // Sends a request for a chunk to the SafeBrowsing servers.
256 void IssueChunkRequest();
258 // Runs the protocol parser on received data and update the
259 // SafeBrowsingService with the new content. Returns 'true' on successful
260 // parse, 'false' on error.
261 bool HandleServiceResponse(const GURL
& url
, const char* data
, size_t length
);
263 // Updates internal state for each GetHash response error, assuming that the
264 // current time is |now|.
265 void HandleGetHashError(const base::Time
& now
);
267 // Helper function for update completion.
268 void UpdateFinished(bool success
);
269 void UpdateFinished(bool success
, bool back_off
);
271 // A callback that runs if we timeout waiting for a response to an update
272 // request. We use this to properly set our update state.
273 void UpdateResponseTimeout();
275 // Called after the chunks are added to the database.
276 void OnAddChunksComplete();
279 // Map of GetHash requests to parameters which created it.
280 struct FullHashDetails
{
282 FullHashDetails(FullHashCallback callback
, bool is_download
);
285 FullHashCallback callback
;
288 typedef base::hash_map
<const net::URLFetcher
*, FullHashDetails
> HashRequests
;
290 // The factory that controls the creation of SafeBrowsingProtocolManager.
291 // This is used by tests.
292 static SBProtocolManagerFactory
* factory_
;
295 SafeBrowsingProtocolManagerDelegate
* delegate_
;
297 // Current active request (in case we need to cancel) for updates or chunks
298 // from the SafeBrowsing service. We can only have one of these outstanding
299 // at any given time unlike GetHash requests, which are tracked separately.
300 scoped_ptr
<net::URLFetcher
> request_
;
302 // The kind of request that is currently in progress.
303 SafeBrowsingRequestType request_type_
;
305 // The number of HTTP response errors, used for request backoff timing.
306 size_t update_error_count_
;
307 size_t gethash_error_count_
;
309 // Multipliers which double (max == 8) for each error after the second.
310 size_t update_back_off_mult_
;
311 size_t gethash_back_off_mult_
;
313 // Multiplier between 0 and 1 to spread clients over an interval.
314 float back_off_fuzz_
;
316 // The list for which we are make a request.
317 std::string list_name_
;
319 // For managing the next earliest time to query the SafeBrowsing servers for
321 base::TimeDelta next_update_interval_
;
322 base::OneShotTimer
<SafeBrowsingProtocolManager
> update_timer_
;
324 // timeout_timer_ is used to interrupt update requests which are taking
326 base::OneShotTimer
<SafeBrowsingProtocolManager
> timeout_timer_
;
328 // All chunk requests that need to be made.
329 std::deque
<ChunkUrl
> chunk_request_urls_
;
331 HashRequests hash_requests_
;
333 // The next scheduled update has special behavior for the first 2 requests.
334 enum UpdateRequestState
{
339 UpdateRequestState update_state_
;
341 // True if the service has been given an add/sub chunk but it hasn't been
342 // added to the database yet.
343 bool chunk_pending_to_write_
;
345 // The last time we successfully received an update.
346 base::Time last_update_
;
348 // While in GetHash backoff, we can't make another GetHash until this time.
349 base::Time next_gethash_time_
;
351 // Current product version sent in each request.
352 std::string version_
;
354 // Used for measuring chunk request latency.
355 base::Time chunk_request_start_
;
357 // Tracks the size of each update (in bytes).
360 // The safe browsing client name sent in each request.
361 std::string client_name_
;
363 // A string that is appended to the end of URLs for download, gethash,
364 // safebrowsing hits and chunk update requests.
365 std::string additional_query_
;
367 // The context we use to issue network requests.
368 scoped_refptr
<net::URLRequestContextGetter
> request_context_getter_
;
370 // URL prefix where browser fetches safebrowsing chunk updates, and hashes.
371 std::string url_prefix_
;
373 // Backup URL prefixes for updates.
374 std::string backup_url_prefixes_
[BACKUP_UPDATE_REASON_MAX
];
376 // The current reason why the backup update request is happening.
377 BackupUpdateReason backup_update_reason_
;
379 // Data to POST when doing an update.
380 std::string update_list_data_
;
382 // When true, protocol manager will not start an update unless
383 // ForceScheduleNextUpdate() is called. This is set for testing purpose.
384 bool disable_auto_update_
;
386 #if defined(OS_ANDROID)
387 // When true, protocol_manager will not check network connection
388 // type when scheduling next update. This is set for testing purpose.
389 bool disable_connection_check_
;
392 // ID for URLFetchers for testing.
395 // Whether the app is in foreground or background.
396 bool app_in_foreground_
;
398 DISALLOW_COPY_AND_ASSIGN(SafeBrowsingProtocolManager
);
401 // Interface of a factory to create ProtocolManager. Useful for tests.
402 class SBProtocolManagerFactory
{
404 SBProtocolManagerFactory() {}
405 virtual ~SBProtocolManagerFactory() {}
406 virtual SafeBrowsingProtocolManager
* CreateProtocolManager(
407 SafeBrowsingProtocolManagerDelegate
* delegate
,
408 net::URLRequestContextGetter
* request_context_getter
,
409 const SafeBrowsingProtocolConfig
& config
) = 0;
411 DISALLOW_COPY_AND_ASSIGN(SBProtocolManagerFactory
);
414 // Delegate interface for the SafeBrowsingProtocolManager.
415 class SafeBrowsingProtocolManagerDelegate
{
417 typedef base::Callback
<void(
418 const std::vector
<SBListChunkRanges
>&, /* List of chunks */
419 bool, /* database_error */
420 bool /* is_extended_reporting */
421 )> GetChunksCallback
;
422 typedef base::Callback
<void(void)> AddChunksCallback
;
424 virtual ~SafeBrowsingProtocolManagerDelegate();
426 // |UpdateStarted()| is called just before the SafeBrowsing update protocol
428 virtual void UpdateStarted() = 0;
430 // |UpdateFinished()| is called just after the SafeBrowsing update protocol
432 virtual void UpdateFinished(bool success
) = 0;
434 // Wipe out the local database. The SafeBrowsing server can request this.
435 virtual void ResetDatabase() = 0;
437 // Retrieve all the local database chunks, and invoke |callback| with the
438 // results. The SafeBrowsingProtocolManagerDelegate must only invoke the
439 // callback if the SafeBrowsingProtocolManager is still alive. Only one call
440 // may be made to GetChunks at a time.
441 virtual void GetChunks(GetChunksCallback callback
) = 0;
443 // Add new chunks to the database. Invokes |callback| when complete, but must
444 // call at a later time.
445 virtual void AddChunks(const std::string
& list
,
446 scoped_ptr
<ScopedVector
<SBChunkData
> > chunks
,
447 AddChunksCallback callback
) = 0;
449 // Delete chunks from the database.
450 virtual void DeleteChunks(
451 scoped_ptr
<std::vector
<SBChunkDelete
> > chunk_deletes
) = 0;
454 #endif // CHROME_BROWSER_SAFE_BROWSING_PROTOCOL_MANAGER_H_