1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // The basic usage of the Filter interface is described in the comment at
6 // the beginning of filter.h. If Filter::Factory is passed a vector of
7 // size greater than 1, that interface is implemented by a series of filters
8 // connected in a chain. In such a case the first filter
9 // in the chain proxies calls to ReadData() so that its return values
10 // apply to the entire chain.
12 // In a filter chain, the data flows from first filter (held by the
13 // caller) down the chain. When ReadData() is called on any filter
14 // except for the last filter, it proxies the call down the chain,
15 // filling in the input buffers of subsequent filters if needed (==
16 // that filter's last_status() value is FILTER_NEED_MORE_DATA) and
17 // available (== the current filter has data it can output). The last
18 // Filter will then output data if possible, and return
19 // FILTER_NEED_MORE_DATA if not. Because the indirection pushes
20 // data along the filter chain at each level if it's available and the
21 // next filter needs it, a return value of FILTER_NEED_MORE_DATA from the
22 // final filter will apply to the entire chain.
24 #include "net/filter/filter.h"
26 #include "base/files/file_path.h"
27 #include "base/strings/string_util.h"
28 #include "net/base/io_buffer.h"
29 #include "net/base/sdch_net_log_params.h"
30 #include "net/filter/gzip_filter.h"
31 #include "net/filter/sdch_filter.h"
32 #include "net/url_request/url_request_context.h"
39 // Filter types (using canonical lower case only):
40 const char kDeflate
[] = "deflate";
41 const char kGZip
[] = "gzip";
42 const char kXGZip
[] = "x-gzip";
43 const char kSdch
[] = "sdch";
44 // compress and x-compress are currently not supported. If we decide to support
45 // them, we'll need the same mime type compatibility hack we have for gzip. For
46 // more information, see Firefox's nsHttpChannel::ProcessNormal.
49 const char kTextHtml
[] = "text/html";
51 // Buffer size allocated when de-compressing data.
52 const int kFilterBufSize
= 32 * 1024;
54 void LogSdchProblem(const FilterContext
& filter_context
,
55 SdchProblemCode problem
) {
56 SdchManager::SdchErrorRecovery(problem
);
57 filter_context
.GetNetLog().AddEvent(
58 NetLog::TYPE_SDCH_DECODING_ERROR
,
59 base::Bind(&NetLogSdchResourceProblemCallback
, problem
));
62 std::string
FilterTypeAsString(Filter::FilterType type_id
) {
64 case Filter::FILTER_TYPE_DEFLATE
:
65 return "FILTER_TYPE_DEFLATE";
66 case Filter::FILTER_TYPE_GZIP
:
67 return "FILTER_TYPE_GZIP";
68 case Filter::FILTER_TYPE_GZIP_HELPING_SDCH
:
69 return "FILTER_TYPE_GZIP_HELPING_SDCH";
70 case Filter::FILTER_TYPE_SDCH
:
71 return "FILTER_TYPE_SDCH";
72 case Filter::FILTER_TYPE_SDCH_POSSIBLE
:
73 return "FILTER_TYPE_SDCH_POSSIBLE ";
74 case Filter::FILTER_TYPE_UNSUPPORTED
:
75 return "FILTER_TYPE_UNSUPPORTED";
82 FilterContext::~FilterContext() {
88 Filter
* Filter::Factory(const std::vector
<FilterType
>& filter_types
,
89 const FilterContext
& filter_context
) {
90 if (filter_types
.empty())
93 Filter
* filter_list
= NULL
; // Linked list of filters.
94 for (size_t i
= 0; i
< filter_types
.size(); i
++) {
95 filter_list
= PrependNewFilter(filter_types
[i
], filter_context
,
96 kFilterBufSize
, filter_list
);
104 Filter
* Filter::GZipFactory() {
105 return InitGZipFilter(FILTER_TYPE_GZIP
, kFilterBufSize
);
109 Filter
* Filter::FactoryForTests(const std::vector
<FilterType
>& filter_types
,
110 const FilterContext
& filter_context
,
112 if (filter_types
.empty())
115 Filter
* filter_list
= NULL
; // Linked list of filters.
116 for (size_t i
= 0; i
< filter_types
.size(); i
++) {
117 filter_list
= PrependNewFilter(filter_types
[i
], filter_context
,
118 buffer_size
, filter_list
);
125 Filter::FilterStatus
Filter::ReadData(char* dest_buffer
, int* dest_len
) {
126 const int dest_buffer_capacity
= *dest_len
;
127 if (last_status_
== FILTER_ERROR
)
129 if (!next_filter_
.get())
130 return last_status_
= ReadFilteredData(dest_buffer
, dest_len
);
132 // This filter needs more data, but it's not clear that the rest of
133 // the chain does; delegate the actual status return to the next filter.
134 if (last_status_
== FILTER_NEED_MORE_DATA
&& !stream_data_len())
135 return next_filter_
->ReadData(dest_buffer
, dest_len
);
138 if (next_filter_
->last_status() == FILTER_NEED_MORE_DATA
) {
139 PushDataIntoNextFilter();
140 if (FILTER_ERROR
== last_status_
)
143 *dest_len
= dest_buffer_capacity
; // Reset the input/output parameter.
144 next_filter_
->ReadData(dest_buffer
, dest_len
);
145 if (FILTER_NEED_MORE_DATA
== last_status_
)
146 return next_filter_
->last_status();
148 // In the case where this filter has data internally, and is indicating such
149 // with a last_status_ of FILTER_OK, but at the same time the next filter in
150 // the chain indicated it FILTER_NEED_MORE_DATA, we have to be cautious
151 // about confusing the caller. The API confusion can appear if we return
152 // FILTER_OK (suggesting we have more data in aggregate), but yet we don't
153 // populate our output buffer. When that is the case, we need to
154 // alternately call our filter element, and the next_filter element until we
155 // get out of this state (by pumping data into the next filter until it
156 // outputs data, or it runs out of data and reports that it NEED_MORE_DATA.)
157 } while (FILTER_OK
== last_status_
&&
158 FILTER_NEED_MORE_DATA
== next_filter_
->last_status() &&
161 if (next_filter_
->last_status() == FILTER_ERROR
)
166 bool Filter::FlushStreamBuffer(int stream_data_len
) {
167 DCHECK_LE(stream_data_len
, stream_buffer_size_
);
168 if (stream_data_len
<= 0 || stream_data_len
> stream_buffer_size_
)
171 DCHECK(stream_buffer());
172 // Bail out if there is more data in the stream buffer to be filtered.
173 if (!stream_buffer() || stream_data_len_
)
176 next_stream_data_
= stream_buffer()->data();
177 stream_data_len_
= stream_data_len
;
178 last_status_
= FILTER_OK
;
183 Filter::FilterType
Filter::ConvertEncodingToType(
184 const std::string
& filter_type
) {
186 if (LowerCaseEqualsASCII(filter_type
, kDeflate
)) {
187 type_id
= FILTER_TYPE_DEFLATE
;
188 } else if (LowerCaseEqualsASCII(filter_type
, kGZip
) ||
189 LowerCaseEqualsASCII(filter_type
, kXGZip
)) {
190 type_id
= FILTER_TYPE_GZIP
;
191 } else if (LowerCaseEqualsASCII(filter_type
, kSdch
)) {
192 type_id
= FILTER_TYPE_SDCH
;
194 // Note we also consider "identity" and "uncompressed" UNSUPPORTED as
195 // filter should be disabled in such cases.
196 type_id
= FILTER_TYPE_UNSUPPORTED
;
202 void Filter::FixupEncodingTypes(
203 const FilterContext
& filter_context
,
204 std::vector
<FilterType
>* encoding_types
) {
205 std::string mime_type
;
206 bool success
= filter_context
.GetMimeType(&mime_type
);
207 DCHECK(success
|| mime_type
.empty());
209 // If the request was for SDCH content, then we might need additional fixups.
210 if (!filter_context
.SdchDictionariesAdvertised()) {
211 // It was not an SDCH request, so we'll just record stats.
212 if (1 < encoding_types
->size()) {
213 // Multiple filters were intended to only be used for SDCH (thus far!)
214 LogSdchProblem(filter_context
, SDCH_MULTIENCODING_FOR_NON_SDCH_REQUEST
);
216 if ((1 == encoding_types
->size()) &&
217 (FILTER_TYPE_SDCH
== encoding_types
->front())) {
218 LogSdchProblem(filter_context
,
219 SDCH_SDCH_CONTENT_ENCODE_FOR_NON_SDCH_REQUEST
);
224 // The request was tagged as an SDCH request, which means the server supplied
225 // a dictionary, and we advertised it in the request. Some proxies will do
226 // very strange things to the request, or the response, so we have to handle
229 // If content encoding included SDCH, then everything is "relatively" fine.
230 if (!encoding_types
->empty() &&
231 (FILTER_TYPE_SDCH
== encoding_types
->front())) {
232 // Some proxies (found currently in Argentina) strip the Content-Encoding
233 // text from "sdch,gzip" to a mere "sdch" without modifying the compressed
234 // payload. To handle this gracefully, we simulate the "probably" deleted
235 // ",gzip" by appending a tentative gzip decode, which will default to a
236 // no-op pass through filter if it doesn't get gzip headers where expected.
237 if (1 == encoding_types
->size()) {
238 encoding_types
->push_back(FILTER_TYPE_GZIP_HELPING_SDCH
);
239 LogSdchProblem(filter_context
, SDCH_OPTIONAL_GUNZIP_ENCODING_ADDED
);
244 // There are now several cases to handle for an SDCH request. Foremost, if
245 // the outbound request was stripped so as not to advertise support for
246 // encodings, we might get back content with no encoding, or (for example)
247 // just gzip. We have to be sure that any changes we make allow for such
248 // minimal coding to work. That issue is why we use TENTATIVE filters if we
249 // add any, as those filters sniff the content, and act as pass-through
250 // filters if headers are not found.
252 // If the outbound GET is not modified, then the server will generally try to
253 // send us SDCH encoded content. As that content returns, there are several
254 // corruptions of the header "content-encoding" that proxies may perform (and
255 // have been detected in the wild). We already dealt with the a honest
256 // content encoding of "sdch,gzip" being corrupted into "sdch" with on change
257 // of the actual content. Another common corruption is to either disscard
258 // the accurate content encoding, or to replace it with gzip only (again, with
259 // no change in actual content). The last observed corruption it to actually
260 // change the content, such as by re-gzipping it, and that may happen along
261 // with corruption of the stated content encoding (wow!).
263 // The one unresolved failure mode comes when we advertise a dictionary, and
264 // the server tries to *send* a gzipped file (not gzip encode content), and
265 // then we could do a gzip decode :-(. Since SDCH is only (currently)
266 // supported server side on paths that only send HTML content, this mode has
267 // never surfaced in the wild (and is unlikely to).
268 // We will gather a lot of stats as we perform the fixups
269 if (StartsWithASCII(mime_type
, kTextHtml
, false)) {
270 // Suspicious case: Advertised dictionary, but server didn't use sdch, and
271 // we're HTML tagged.
272 if (encoding_types
->empty()) {
273 LogSdchProblem(filter_context
, SDCH_ADDED_CONTENT_ENCODING
);
274 } else if (1 == encoding_types
->size()) {
275 LogSdchProblem(filter_context
, SDCH_FIXED_CONTENT_ENCODING
);
277 LogSdchProblem(filter_context
, SDCH_FIXED_CONTENT_ENCODINGS
);
280 // Remarkable case!?! We advertised an SDCH dictionary, content-encoding
281 // was not marked for SDCH processing: Why did the server suggest an SDCH
282 // dictionary in the first place??. Also, the content isn't
283 // tagged as HTML, despite the fact that SDCH encoding is mostly likely for
284 // HTML: Did some anti-virus system strip this tag (sometimes they strip
285 // accept-encoding headers on the request)?? Does the content encoding not
286 // start with "text/html" for some other reason?? We'll report this as a
287 // fixup to a binary file, but it probably really is text/html (some how).
288 if (encoding_types
->empty()) {
289 LogSdchProblem(filter_context
, SDCH_BINARY_ADDED_CONTENT_ENCODING
);
290 } else if (1 == encoding_types
->size()) {
291 LogSdchProblem(filter_context
, SDCH_BINARY_FIXED_CONTENT_ENCODING
);
293 LogSdchProblem(filter_context
, SDCH_BINARY_FIXED_CONTENT_ENCODINGS
);
297 // Leave the existing encoding type to be processed first, and add our
298 // tentative decodings to be done afterwards. Vodaphone UK reportedyl will
299 // perform a second layer of gzip encoding atop the server's sdch,gzip
300 // encoding, and then claim that the content encoding is a mere gzip. As a
301 // result we'll need (in that case) to do the gunzip, plus our tentative
302 // gunzip and tentative SDCH decoding.
303 // This approach nicely handles the empty() list as well, and should work with
304 // other (as yet undiscovered) proxies the choose to re-compressed with some
305 // other encoding (such as bzip2, etc.).
306 encoding_types
->insert(encoding_types
->begin(),
307 FILTER_TYPE_GZIP_HELPING_SDCH
);
308 encoding_types
->insert(encoding_types
->begin(), FILTER_TYPE_SDCH_POSSIBLE
);
312 std::string
Filter::OrderedFilterList() const {
314 return FilterTypeAsString(type_id_
) + "," +
315 next_filter_
->OrderedFilterList();
317 return FilterTypeAsString(type_id_
);
321 Filter::Filter(FilterType type_id
)
322 : stream_buffer_(NULL
),
323 stream_buffer_size_(0),
324 next_stream_data_(NULL
),
326 last_status_(FILTER_NEED_MORE_DATA
),
329 Filter::FilterStatus
Filter::CopyOut(char* dest_buffer
, int* dest_len
) {
331 int input_len
= *dest_len
;
334 if (0 == stream_data_len_
)
335 return Filter::FILTER_NEED_MORE_DATA
;
337 out_len
= std::min(input_len
, stream_data_len_
);
338 memcpy(dest_buffer
, next_stream_data_
, out_len
);
339 *dest_len
+= out_len
;
340 stream_data_len_
-= out_len
;
341 if (0 == stream_data_len_
) {
342 next_stream_data_
= NULL
;
343 return Filter::FILTER_NEED_MORE_DATA
;
345 next_stream_data_
+= out_len
;
346 return Filter::FILTER_OK
;
351 Filter
* Filter::InitGZipFilter(FilterType type_id
, int buffer_size
) {
352 scoped_ptr
<GZipFilter
> gz_filter(new GZipFilter(type_id
));
353 gz_filter
->InitBuffer(buffer_size
);
354 return gz_filter
->InitDecoding(type_id
) ? gz_filter
.release() : NULL
;
358 Filter
* Filter::InitSdchFilter(FilterType type_id
,
359 const FilterContext
& filter_context
,
361 scoped_ptr
<SdchFilter
> sdch_filter(new SdchFilter(type_id
, filter_context
));
362 sdch_filter
->InitBuffer(buffer_size
);
363 return sdch_filter
->InitDecoding(type_id
) ? sdch_filter
.release() : NULL
;
367 Filter
* Filter::PrependNewFilter(FilterType type_id
,
368 const FilterContext
& filter_context
,
370 Filter
* filter_list
) {
371 scoped_ptr
<Filter
> first_filter
; // Soon to be start of chain.
373 case FILTER_TYPE_GZIP_HELPING_SDCH
:
374 case FILTER_TYPE_DEFLATE
:
375 case FILTER_TYPE_GZIP
:
376 first_filter
.reset(InitGZipFilter(type_id
, buffer_size
));
378 case FILTER_TYPE_SDCH
:
379 case FILTER_TYPE_SDCH_POSSIBLE
:
380 if (filter_context
.GetURLRequestContext()->sdch_manager() &&
381 SdchManager::sdch_enabled()) {
383 InitSdchFilter(type_id
, filter_context
, buffer_size
));
390 if (!first_filter
.get())
393 first_filter
->next_filter_
.reset(filter_list
);
394 return first_filter
.release();
397 void Filter::InitBuffer(int buffer_size
) {
398 DCHECK(!stream_buffer());
399 DCHECK_GT(buffer_size
, 0);
400 stream_buffer_
= new IOBuffer(buffer_size
);
401 stream_buffer_size_
= buffer_size
;
404 void Filter::PushDataIntoNextFilter() {
405 IOBuffer
* next_buffer
= next_filter_
->stream_buffer();
406 int next_size
= next_filter_
->stream_buffer_size();
407 last_status_
= ReadFilteredData(next_buffer
->data(), &next_size
);
408 if (FILTER_ERROR
!= last_status_
)
409 next_filter_
->FlushStreamBuffer(next_size
);