1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "net/filter/sdch_filter.h"
12 #include "base/logging.h"
13 #include "base/metrics/histogram.h"
14 #include "base/values.h"
15 #include "net/base/sdch_manager.h"
16 #include "net/base/sdch_net_log_params.h"
17 #include "net/base/sdch_problem_codes.h"
18 #include "net/url_request/url_request_context.h"
20 #include "sdch/open-vcdiff/src/google/vcdecoder.h"
26 // Disambiguate various types of responses that trigger a meta-refresh,
27 // failure, or fallback to pass-through.
28 enum ResponseCorruptionDetectionCause
{
31 // 404 Http Response Code
34 // Not a 200 Http Response Code
37 // Cached before dictionary retrieved.
38 RESPONSE_OLD_UNENCODED
= 3,
40 // Speculative but incorrect SDCH filtering was added added.
41 RESPONSE_TENTATIVE_SDCH
= 4,
43 // Missing correct dict for decoding.
44 RESPONSE_NO_DICTIONARY
= 5,
46 // Not an SDCH response but should be.
47 RESPONSE_CORRUPT_SDCH
= 6,
49 // No dictionary was advertised with the request, the server claims
50 // to have encoded with SDCH anyway, but it isn't an SDCH response.
51 RESPONSE_ENCODING_LIE
= 7,
56 const char* ResponseCorruptionDetectionCauseToString(
57 ResponseCorruptionDetectionCause cause
) {
58 const char* cause_string
= "<unknown>";
61 cause_string
= "NONE";
66 case RESPONSE_NOT_200
:
67 cause_string
= "NOT_200";
69 case RESPONSE_OLD_UNENCODED
:
70 cause_string
= "OLD_UNENCODED";
72 case RESPONSE_TENTATIVE_SDCH
:
73 cause_string
= "TENTATIVE_SDCH";
75 case RESPONSE_NO_DICTIONARY
:
76 cause_string
= "NO_DICTIONARY";
78 case RESPONSE_CORRUPT_SDCH
:
79 cause_string
= "CORRUPT_SDCH";
81 case RESPONSE_ENCODING_LIE
:
82 cause_string
= "ENCODING_LIE";
85 cause_string
= "<Error: max enum value>";
91 base::Value
* NetLogSdchResponseCorruptionDetectionCallback(
92 ResponseCorruptionDetectionCause cause
,
94 NetLog::LogLevel log_level
) {
95 base::DictionaryValue
* dict
= new base::DictionaryValue();
96 dict
->SetString("cause", ResponseCorruptionDetectionCauseToString(cause
));
97 dict
->SetBoolean("cached", cached
);
103 SdchFilter::SdchFilter(FilterType type
, const FilterContext
& filter_context
)
105 filter_context_(filter_context
),
106 decoding_status_(DECODING_UNINITIALIZED
),
108 dictionary_hash_is_plausible_(false),
110 url_request_context_(filter_context
.GetURLRequestContext()),
111 dest_buffer_excess_(),
112 dest_buffer_excess_index_(0),
115 possible_pass_through_(false) {
116 bool success
= filter_context
.GetMimeType(&mime_type_
);
118 success
= filter_context
.GetURL(&url_
);
120 DCHECK(url_request_context_
->sdch_manager());
123 SdchFilter::~SdchFilter() {
124 // All code here is for gathering stats, and can be removed when SDCH is
125 // considered stable.
127 static int filter_use_count
= 0;
129 if (META_REFRESH_RECOVERY
== decoding_status_
) {
130 UMA_HISTOGRAM_COUNTS("Sdch3.FilterUseBeforeDisabling", filter_use_count
);
133 if (vcdiff_streaming_decoder_
.get()) {
134 if (!vcdiff_streaming_decoder_
->FinishDecoding()) {
135 decoding_status_
= DECODING_ERROR
;
136 LogSdchProblem(SDCH_INCOMPLETE_SDCH_CONTENT
);
137 // Make it possible for the user to hit reload, and get non-sdch content.
138 // Note this will "wear off" quickly enough, and is just meant to assure
139 // in some rare case that the user is not stuck.
140 url_request_context_
->sdch_manager()->BlacklistDomain(
141 url_
, SDCH_INCOMPLETE_SDCH_CONTENT
);
142 UMA_HISTOGRAM_COUNTS("Sdch3.PartialBytesIn",
143 static_cast<int>(filter_context_
.GetByteReadCount()));
144 UMA_HISTOGRAM_COUNTS("Sdch3.PartialVcdiffIn", source_bytes_
);
145 UMA_HISTOGRAM_COUNTS("Sdch3.PartialVcdiffOut", output_bytes_
);
149 if (!dest_buffer_excess_
.empty()) {
150 // Filter chaining error, or premature teardown.
151 LogSdchProblem(SDCH_UNFLUSHED_CONTENT
);
152 UMA_HISTOGRAM_COUNTS("Sdch3.UnflushedBytesIn",
153 static_cast<int>(filter_context_
.GetByteReadCount()));
154 UMA_HISTOGRAM_COUNTS("Sdch3.UnflushedBufferSize",
155 dest_buffer_excess_
.size());
156 UMA_HISTOGRAM_COUNTS("Sdch3.UnflushedVcdiffIn", source_bytes_
);
157 UMA_HISTOGRAM_COUNTS("Sdch3.UnflushedVcdiffOut", output_bytes_
);
160 if (filter_context_
.IsCachedContent()) {
161 // Not a real error, but it is useful to have this tally.
162 // TODO(jar): Remove this stat after SDCH stability is validated.
163 LogSdchProblem(SDCH_CACHE_DECODED
);
164 return; // We don't need timing stats, and we aready got ratios.
167 switch (decoding_status_
) {
168 case DECODING_IN_PROGRESS
: {
170 UMA_HISTOGRAM_PERCENTAGE("Sdch3.Network_Decode_Ratio_a",
172 (filter_context_
.GetByteReadCount() * 100) / output_bytes_
));
173 UMA_HISTOGRAM_COUNTS("Sdch3.Network_Decode_Bytes_VcdiffOut_a",
175 filter_context_
.RecordPacketStats(FilterContext::SDCH_DECODE
);
177 // Allow latency experiments to proceed.
178 url_request_context_
->sdch_manager()->SetAllowLatencyExperiment(
181 // Notify successful dictionary usage.
182 url_request_context_
->sdch_manager()->OnDictionaryUsed(
183 dictionary_
->server_hash());
188 filter_context_
.RecordPacketStats(FilterContext::SDCH_PASSTHROUGH
);
191 case DECODING_UNINITIALIZED
: {
192 LogSdchProblem(SDCH_UNINITIALIZED
);
195 case WAITING_FOR_DICTIONARY_SELECTION
: {
196 LogSdchProblem(SDCH_PRIOR_TO_DICTIONARY
);
199 case DECODING_ERROR
: {
200 LogSdchProblem(SDCH_DECODE_ERROR
);
203 case META_REFRESH_RECOVERY
: {
204 // Already accounted for when set.
210 bool SdchFilter::InitDecoding(Filter::FilterType filter_type
) {
211 if (decoding_status_
!= DECODING_UNINITIALIZED
)
214 // Handle case where sdch filter is guessed, but not required.
215 if (FILTER_TYPE_SDCH_POSSIBLE
== filter_type
)
216 possible_pass_through_
= true;
218 // Initialize decoder only after we have a dictionary in hand.
219 decoding_status_
= WAITING_FOR_DICTIONARY_SELECTION
;
224 static const char* kDecompressionErrorHtml
=
225 "<head><META HTTP-EQUIV=\"Refresh\" CONTENT=\"0\"></head>"
226 "<div style=\"position:fixed;top:0;left:0;width:100%;border-width:thin;"
227 "border-color:black;border-style:solid;text-align:left;font-family:arial;"
228 "font-size:10pt;foreground-color:black;background-color:white\">"
229 "An error occurred. This page will be reloaded shortly. "
230 "Or press the \"reload\" button now to reload it immediately."
233 static const char* kDecompressionErrorHtml
=
234 "<head><META HTTP-EQUIV=\"Refresh\" CONTENT=\"0\"></head>";
237 Filter::FilterStatus
SdchFilter::ReadFilteredData(char* dest_buffer
,
239 int available_space
= *dest_len
;
240 *dest_len
= 0; // Nothing output yet.
242 if (!dest_buffer
|| available_space
<= 0)
245 if (WAITING_FOR_DICTIONARY_SELECTION
== decoding_status_
) {
246 FilterStatus status
= InitializeDictionary();
247 if (FILTER_NEED_MORE_DATA
== status
)
248 return FILTER_NEED_MORE_DATA
;
249 if (FILTER_ERROR
== status
) {
250 DCHECK_EQ(DECODING_ERROR
, decoding_status_
);
251 DCHECK_EQ(0u, dest_buffer_excess_index_
);
252 DCHECK(dest_buffer_excess_
.empty());
253 // This is where we try very hard to do error recovery, and make this
254 // protocol robust in the face of proxies that do many different things.
255 // If we decide that things are looking very bad (too hard to recover),
256 // we may even issue a "meta-refresh" to reload the page without an SDCH
257 // advertisement (so that we are sure we're not hurting anything).
259 // Watch out for an error page inserted by the proxy as part of a 40x
260 // error response. When we see such content molestation, we certainly
261 // need to fall into the meta-refresh case.
262 ResponseCorruptionDetectionCause cause
= RESPONSE_NONE
;
263 if (filter_context_
.GetResponseCode() == 404) {
264 // We could be more generous, but for now, only a "NOT FOUND" code will
265 // cause a pass through. All other bad codes will fall into a
267 LogSdchProblem(SDCH_PASS_THROUGH_404_CODE
);
268 cause
= RESPONSE_404
;
269 decoding_status_
= PASS_THROUGH
;
270 } else if (filter_context_
.GetResponseCode() != 200) {
271 // We need to meta-refresh, with SDCH disabled.
272 cause
= RESPONSE_NOT_200
;
273 } else if (filter_context_
.IsCachedContent()
274 && !dictionary_hash_is_plausible_
) {
275 // We must have hit the back button, and gotten content that was fetched
276 // before we *really* advertised SDCH and a dictionary.
277 LogSdchProblem(SDCH_PASS_THROUGH_OLD_CACHED
);
278 decoding_status_
= PASS_THROUGH
;
279 cause
= RESPONSE_OLD_UNENCODED
;
280 } else if (possible_pass_through_
) {
281 // This is the potentially most graceful response. There really was no
282 // error. We were just overly cautious when we added a TENTATIVE_SDCH.
283 // We added the sdch coding tag, and it should not have been added.
284 // This can happen in server experiments, where the server decides
285 // not to use sdch, even though there is a dictionary. To be
286 // conservative, we locally added the tentative sdch (fearing that a
287 // proxy stripped it!) and we must now recant (pass through).
289 // However.... just to be sure we don't get burned by proxies that
290 // re-compress with gzip or other system, we can sniff to see if this
291 // is compressed data etc. For now, we do nothing, which gets us into
292 // the meta-refresh result.
293 // TODO(jar): Improve robustness by sniffing for valid text that we can
294 // actual use re: decoding_status_ = PASS_THROUGH;
295 cause
= RESPONSE_TENTATIVE_SDCH
;
296 } else if (dictionary_hash_is_plausible_
) {
297 // We need a meta-refresh since we don't have the dictionary.
298 // The common cause is a restart of the browser, where we try to render
299 // cached content that was saved when we had a dictionary.
300 cause
= RESPONSE_NO_DICTIONARY
;
301 } else if (filter_context_
.SdchDictionariesAdvertised()) {
302 // This is a very corrupt SDCH request response. We can't decode it.
303 // We'll use a meta-refresh, and get content without asking for SDCH.
304 // This will also progressively disable SDCH for this domain.
305 cause
= RESPONSE_CORRUPT_SDCH
;
307 // One of the first 9 bytes precluded consideration as a hash.
308 // This can't be an SDCH payload, even though the server said it was.
309 // This is a major error, as the server or proxy tagged this SDCH even
311 // Meta-refresh won't help, as we didn't advertise an SDCH dictionary!!
312 // Worse yet, meta-refresh could lead to an infinite refresh loop.
313 LogSdchProblem(SDCH_PASSING_THROUGH_NON_SDCH
);
314 decoding_status_
= PASS_THROUGH
;
315 // ... but further back-off on advertising SDCH support.
316 url_request_context_
->sdch_manager()->BlacklistDomain(
317 url_
, SDCH_PASSING_THROUGH_NON_SDCH
);
318 cause
= RESPONSE_ENCODING_LIE
;
320 DCHECK_NE(RESPONSE_NONE
, cause
);
322 // Use if statement rather than ?: because UMA_HISTOGRAM_ENUMERATION
323 // caches the histogram name based on the call site.
324 if (filter_context_
.IsCachedContent()) {
325 UMA_HISTOGRAM_ENUMERATION(
326 "Sdch3.ResponseCorruptionDetection.Cached", cause
, RESPONSE_MAX
);
328 UMA_HISTOGRAM_ENUMERATION(
329 "Sdch3.ResponseCorruptionDetection.Uncached", cause
, RESPONSE_MAX
);
331 filter_context_
.GetNetLog().AddEvent(
332 NetLog::TYPE_SDCH_RESPONSE_CORRUPTION_DETECTION
,
333 base::Bind(&NetLogSdchResponseCorruptionDetectionCallback
, cause
,
334 filter_context_
.IsCachedContent()));
336 if (decoding_status_
== PASS_THROUGH
) {
337 dest_buffer_excess_
= dictionary_hash_
; // Send what we scanned.
339 // This is where we try to do the expensive meta-refresh.
340 if (std::string::npos
== mime_type_
.find("text/html")) {
341 // Since we can't do a meta-refresh (along with an exponential
342 // backoff), we'll just make sure this NEVER happens again.
343 SdchProblemCode problem
= (filter_context_
.IsCachedContent()
344 ? SDCH_CACHED_META_REFRESH_UNSUPPORTED
345 : SDCH_META_REFRESH_UNSUPPORTED
);
346 url_request_context_
->sdch_manager()->BlacklistDomainForever(
348 LogSdchProblem(problem
);
351 // HTML content means we can issue a meta-refresh, and get the content
352 // again, perhaps without SDCH (to be safe).
353 if (filter_context_
.IsCachedContent()) {
354 // Cached content is probably a startup tab, so we'll just get fresh
355 // content and try again, without disabling sdch.
356 LogSdchProblem(SDCH_META_REFRESH_CACHED_RECOVERY
);
358 // Since it wasn't in the cache, we definately need at least some
359 // period of blacklisting to get the correct content.
360 url_request_context_
->sdch_manager()->BlacklistDomain(
361 url_
, SDCH_META_REFRESH_RECOVERY
);
362 LogSdchProblem(SDCH_META_REFRESH_RECOVERY
);
364 decoding_status_
= META_REFRESH_RECOVERY
;
365 // Issue a meta redirect with SDCH disabled.
366 dest_buffer_excess_
= kDecompressionErrorHtml
;
369 DCHECK_EQ(DECODING_IN_PROGRESS
, decoding_status_
);
373 int amount
= OutputBufferExcess(dest_buffer
, available_space
);
375 dest_buffer
+= amount
;
376 available_space
-= amount
;
377 DCHECK_GE(available_space
, 0);
379 if (available_space
<= 0)
381 DCHECK(dest_buffer_excess_
.empty());
382 DCHECK_EQ(0u, dest_buffer_excess_index_
);
384 if (decoding_status_
!= DECODING_IN_PROGRESS
) {
385 if (META_REFRESH_RECOVERY
== decoding_status_
) {
386 // Absorb all input data. We've already output page reload HTML.
387 next_stream_data_
= NULL
;
388 stream_data_len_
= 0;
389 return FILTER_NEED_MORE_DATA
;
391 if (PASS_THROUGH
== decoding_status_
) {
392 // We must pass in available_space, but it will be changed to bytes_used.
393 FilterStatus result
= CopyOut(dest_buffer
, &available_space
);
394 // Accumulate the returned count of bytes_used (a.k.a., available_space).
395 *dest_len
+= available_space
;
399 decoding_status_
= DECODING_ERROR
;
403 if (!next_stream_data_
|| stream_data_len_
<= 0)
404 return FILTER_NEED_MORE_DATA
;
406 bool ret
= vcdiff_streaming_decoder_
->DecodeChunk(
407 next_stream_data_
, stream_data_len_
, &dest_buffer_excess_
);
408 // Assume all data was used in decoding.
409 next_stream_data_
= NULL
;
410 source_bytes_
+= stream_data_len_
;
411 stream_data_len_
= 0;
412 output_bytes_
+= dest_buffer_excess_
.size();
414 vcdiff_streaming_decoder_
.reset(NULL
); // Don't call it again.
415 decoding_status_
= DECODING_ERROR
;
416 LogSdchProblem(SDCH_DECODE_BODY_ERROR
);
420 amount
= OutputBufferExcess(dest_buffer
, available_space
);
422 dest_buffer
+= amount
;
423 available_space
-= amount
;
424 if (0 == available_space
&& !dest_buffer_excess_
.empty())
426 return FILTER_NEED_MORE_DATA
;
429 Filter::FilterStatus
SdchFilter::InitializeDictionary() {
430 const size_t kServerIdLength
= 9; // Dictionary hash plus null from server.
431 size_t bytes_needed
= kServerIdLength
- dictionary_hash_
.size();
432 DCHECK_GT(bytes_needed
, 0u);
433 if (!next_stream_data_
)
434 return FILTER_NEED_MORE_DATA
;
435 if (static_cast<size_t>(stream_data_len_
) < bytes_needed
) {
436 dictionary_hash_
.append(next_stream_data_
, stream_data_len_
);
437 next_stream_data_
= NULL
;
438 stream_data_len_
= 0;
439 return FILTER_NEED_MORE_DATA
;
441 dictionary_hash_
.append(next_stream_data_
, bytes_needed
);
442 DCHECK(kServerIdLength
== dictionary_hash_
.size());
443 stream_data_len_
-= bytes_needed
;
444 DCHECK_LE(0, stream_data_len_
);
445 if (stream_data_len_
> 0)
446 next_stream_data_
+= bytes_needed
;
448 next_stream_data_
= NULL
;
450 DCHECK(!dictionary_
);
451 dictionary_hash_is_plausible_
= true; // Assume plausible, but check.
453 SdchProblemCode rv
= SDCH_OK
;
454 if ('\0' == dictionary_hash_
[kServerIdLength
- 1]) {
455 std::string
server_hash(dictionary_hash_
, 0, kServerIdLength
- 1);
456 SdchManager::DictionarySet
* handle
=
457 filter_context_
.SdchDictionariesAdvertised();
459 dictionary_
= handle
->GetDictionary(server_hash
);
461 // This is a hack. Naively, the dictionaries available for
462 // decoding should be only the ones advertised. However, there are
463 // cases, specifically resources encoded with old dictionaries living
464 // in the cache, that mean the full set of dictionaries should be made
465 // available for decoding. It's not known how often this happens;
466 // if it happens rarely enough, this code can be removed.
468 // TODO(rdsmith): Long-term, a better solution is necessary, since
469 // an entry in the cache being encoded with the dictionary doesn't
470 // guarantee that the dictionary is present. That solution probably
471 // involves storing unencoded resources in the cache, but might
472 // involve evicting encoded resources on dictionary removal.
473 // See http://crbug.com/383405.
474 unexpected_dictionary_handle_
=
475 url_request_context_
->sdch_manager()->GetDictionarySetByHash(
476 url_
, server_hash
, &rv
);
477 if (unexpected_dictionary_handle_
) {
478 dictionary_
= unexpected_dictionary_handle_
->GetDictionary(server_hash
);
479 // Override SDCH_OK rv; this is still worth logging.
480 rv
= (filter_context_
.IsCachedContent() ?
481 SDCH_UNADVERTISED_DICTIONARY_USED_CACHED
:
482 SDCH_UNADVERTISED_DICTIONARY_USED
);
484 // Since dictionary was not found, check to see if hash was
486 DCHECK(dictionary_hash_
.size() == kServerIdLength
);
487 rv
= SDCH_DICTIONARY_HASH_NOT_FOUND
;
488 for (size_t i
= 0; i
< kServerIdLength
- 1; ++i
) {
489 char base64_char
= dictionary_hash_
[i
];
490 if (!isalnum(base64_char
) &&
491 '-' != base64_char
&& '_' != base64_char
) {
492 dictionary_hash_is_plausible_
= false;
493 rv
= SDCH_DICTIONARY_HASH_MALFORMED
;
500 dictionary_hash_is_plausible_
= false;
501 rv
= SDCH_DICTIONARY_HASH_MALFORMED
;
508 decoding_status_
= DECODING_ERROR
;
512 vcdiff_streaming_decoder_
.reset(new open_vcdiff::VCDiffStreamingDecoder
);
513 vcdiff_streaming_decoder_
->SetAllowVcdTarget(false);
514 vcdiff_streaming_decoder_
->StartDecoding(dictionary_
->text().data(),
515 dictionary_
->text().size());
516 decoding_status_
= DECODING_IN_PROGRESS
;
520 int SdchFilter::OutputBufferExcess(char* const dest_buffer
,
521 size_t available_space
) {
522 if (dest_buffer_excess_
.empty())
524 DCHECK(dest_buffer_excess_
.size() > dest_buffer_excess_index_
);
525 size_t amount
= std::min(available_space
,
526 dest_buffer_excess_
.size() - dest_buffer_excess_index_
);
527 memcpy(dest_buffer
, dest_buffer_excess_
.data() + dest_buffer_excess_index_
,
529 dest_buffer_excess_index_
+= amount
;
530 if (dest_buffer_excess_
.size() <= dest_buffer_excess_index_
) {
531 DCHECK(dest_buffer_excess_
.size() == dest_buffer_excess_index_
);
532 dest_buffer_excess_
.clear();
533 dest_buffer_excess_index_
= 0;
538 void SdchFilter::LogSdchProblem(SdchProblemCode problem
) {
539 SdchManager::SdchErrorRecovery(problem
);
540 filter_context_
.GetNetLog().AddEvent(
541 NetLog::TYPE_SDCH_DECODING_ERROR
,
542 base::Bind(&NetLogSdchResourceProblemCallback
, problem
));