1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "storage/browser/blob/blob_url_request_job.h"
12 #include "base/basictypes.h"
13 #include "base/bind.h"
14 #include "base/compiler_specific.h"
15 #include "base/files/file_util_proxy.h"
16 #include "base/format_macros.h"
17 #include "base/message_loop/message_loop.h"
18 #include "base/numerics/safe_conversions.h"
19 #include "base/stl_util.h"
20 #include "base/strings/string_number_conversions.h"
21 #include "base/strings/stringprintf.h"
22 #include "base/trace_event/trace_event.h"
23 #include "net/base/io_buffer.h"
24 #include "net/base/net_errors.h"
25 #include "net/disk_cache/disk_cache.h"
26 #include "net/http/http_request_headers.h"
27 #include "net/http/http_response_headers.h"
28 #include "net/http/http_response_info.h"
29 #include "net/http/http_util.h"
30 #include "net/url_request/url_request.h"
31 #include "net/url_request/url_request_context.h"
32 #include "net/url_request/url_request_error_job.h"
33 #include "net/url_request/url_request_status.h"
34 #include "storage/browser/fileapi/file_stream_reader.h"
35 #include "storage/browser/fileapi/file_system_context.h"
36 #include "storage/browser/fileapi/file_system_url.h"
37 #include "storage/common/data_element.h"
43 bool IsFileType(DataElement::Type type
) {
45 case DataElement::TYPE_FILE
:
46 case DataElement::TYPE_FILE_FILESYSTEM
:
55 BlobURLRequestJob::BlobURLRequestJob(
56 net::URLRequest
* request
,
57 net::NetworkDelegate
* network_delegate
,
58 scoped_ptr
<BlobDataSnapshot
> blob_data
,
59 storage::FileSystemContext
* file_system_context
,
60 base::SingleThreadTaskRunner
* file_task_runner
)
61 : net::URLRequestJob(request
, network_delegate
),
62 blob_data_(blob_data
.Pass()),
63 file_system_context_(file_system_context
),
64 file_task_runner_(file_task_runner
),
67 pending_get_file_info_count_(0),
68 current_item_index_(0),
69 current_item_offset_(0),
71 byte_range_set_(false),
73 TRACE_EVENT_ASYNC_BEGIN1("Blob", "BlobRequest", this, "uuid",
74 blob_data_
? blob_data_
->uuid() : "NotFound");
75 DCHECK(file_task_runner_
.get());
78 void BlobURLRequestJob::Start() {
79 // Continue asynchronously.
80 base::MessageLoop::current()->PostTask(
82 base::Bind(&BlobURLRequestJob::DidStart
, weak_factory_
.GetWeakPtr()));
85 void BlobURLRequestJob::Kill() {
86 DeleteCurrentFileReader();
88 net::URLRequestJob::Kill();
89 weak_factory_
.InvalidateWeakPtrs();
92 bool BlobURLRequestJob::ReadRawData(net::IOBuffer
* dest
,
95 DCHECK_NE(dest_size
, 0);
97 DCHECK_GE(remaining_bytes_
, 0);
99 // Bail out immediately if we encounter an error.
105 if (remaining_bytes_
< dest_size
)
106 dest_size
= static_cast<int>(remaining_bytes_
);
108 // If we should copy zero bytes because |remaining_bytes_| is zero, short
115 // Keep track of the buffer.
116 DCHECK(!read_buf_
.get());
117 read_buf_
= new net::DrainableIOBuffer(dest
, dest_size
);
119 return ReadLoop(bytes_read
);
122 bool BlobURLRequestJob::GetMimeType(std::string
* mime_type
) const {
126 return response_info_
->headers
->GetMimeType(mime_type
);
129 void BlobURLRequestJob::GetResponseInfo(net::HttpResponseInfo
* info
) {
131 *info
= *response_info_
;
134 int BlobURLRequestJob::GetResponseCode() const {
138 return response_info_
->headers
->response_code();
141 void BlobURLRequestJob::SetExtraRequestHeaders(
142 const net::HttpRequestHeaders
& headers
) {
143 std::string range_header
;
144 if (headers
.GetHeader(net::HttpRequestHeaders::kRange
, &range_header
)) {
145 // We only care about "Range" header here.
146 std::vector
<net::HttpByteRange
> ranges
;
147 if (net::HttpUtil::ParseRangeHeader(range_header
, &ranges
)) {
148 if (ranges
.size() == 1) {
149 byte_range_set_
= true;
150 byte_range_
= ranges
[0];
152 // We don't support multiple range requests in one single URL request,
153 // because we need to do multipart encoding here.
154 // TODO(jianli): Support multipart byte range requests.
155 NotifyFailure(net::ERR_REQUEST_RANGE_NOT_SATISFIABLE
);
161 BlobURLRequestJob::~BlobURLRequestJob() {
162 STLDeleteValues(&index_to_reader_
);
163 TRACE_EVENT_ASYNC_END1("Blob", "BlobRequest", this, "uuid",
164 blob_data_
? blob_data_
->uuid() : "NotFound");
167 void BlobURLRequestJob::DidStart() {
168 current_file_chunk_number_
= 0;
171 // We only support GET request per the spec.
172 if (request()->method() != "GET") {
173 NotifyFailure(net::ERR_METHOD_NOT_SUPPORTED
);
177 // If the blob data is not present, bail out.
179 NotifyFailure(net::ERR_FILE_NOT_FOUND
);
186 bool BlobURLRequestJob::AddItemLength(size_t index
, int64 item_length
) {
187 if (item_length
> kint64max
- total_size_
) {
188 TRACE_EVENT_ASYNC_END1("Blob", "BlobRequest::CountSize", this, "uuid",
190 NotifyFailure(net::ERR_FAILED
);
194 // Cache the size and add it to the total size.
195 DCHECK_LT(index
, item_length_list_
.size());
196 item_length_list_
[index
] = item_length
;
197 total_size_
+= item_length
;
201 bool BlobURLRequestJob::CountSize() {
202 TRACE_EVENT_ASYNC_BEGIN1("Blob", "BlobRequest::CountSize", this, "uuid",
204 pending_get_file_info_count_
= 0;
206 const auto& items
= blob_data_
->items();
207 item_length_list_
.resize(items
.size());
209 for (size_t i
= 0; i
< items
.size(); ++i
) {
210 const BlobDataItem
& item
= *items
.at(i
);
211 if (IsFileType(item
.type())) {
212 ++pending_get_file_info_count_
;
213 storage::FileStreamReader
* const reader
= GetFileStreamReader(i
);
215 NotifyFailure(net::ERR_FAILED
);
218 if (!reader
->GetLength(
219 base::Bind(&BlobURLRequestJob::DidGetFileItemLength
,
220 weak_factory_
.GetWeakPtr(), i
))) {
221 NotifyFailure(net::ERR_FILE_NOT_FOUND
);
227 if (!AddItemLength(i
, item
.length()))
231 if (pending_get_file_info_count_
== 0)
232 DidCountSize(net::OK
);
237 void BlobURLRequestJob::DidCountSize(int error
) {
239 TRACE_EVENT_ASYNC_END1("Blob", "BlobRequest::CountSize", this, "uuid",
242 // If an error occured, bail out.
243 if (error
!= net::OK
) {
244 NotifyFailure(error
);
248 // Apply the range requirement.
249 if (!byte_range_
.ComputeBounds(total_size_
)) {
250 NotifyFailure(net::ERR_REQUEST_RANGE_NOT_SATISFIABLE
);
254 remaining_bytes_
= base::checked_cast
<int64
>(
255 byte_range_
.last_byte_position() - byte_range_
.first_byte_position() + 1);
256 DCHECK_GE(remaining_bytes_
, 0);
258 // Do the seek at the beginning of the request.
259 if (byte_range_
.first_byte_position())
260 Seek(byte_range_
.first_byte_position());
265 void BlobURLRequestJob::DidGetFileItemLength(size_t index
, int64 result
) {
266 // Do nothing if we have encountered an error.
270 if (result
== net::ERR_UPLOAD_FILE_CHANGED
) {
271 NotifyFailure(net::ERR_FILE_NOT_FOUND
);
273 } else if (result
< 0) {
274 NotifyFailure(result
);
278 const auto& items
= blob_data_
->items();
279 DCHECK_LT(index
, items
.size());
280 const BlobDataItem
& item
= *items
.at(index
);
281 DCHECK(IsFileType(item
.type()));
283 uint64 file_length
= result
;
284 uint64 item_offset
= item
.offset();
285 uint64 item_length
= item
.length();
287 if (item_offset
> file_length
) {
288 NotifyFailure(net::ERR_FILE_NOT_FOUND
);
292 uint64 max_length
= file_length
- item_offset
;
294 // If item length is undefined, then we need to use the file size being
295 // resolved in the real time.
296 if (item_length
== std::numeric_limits
<uint64
>::max()) {
297 item_length
= max_length
;
298 } else if (item_length
> max_length
) {
299 NotifyFailure(net::ERR_FILE_NOT_FOUND
);
303 if (!AddItemLength(index
, item_length
))
306 if (--pending_get_file_info_count_
== 0)
307 DidCountSize(net::OK
);
310 void BlobURLRequestJob::Seek(int64 offset
) {
311 // Skip the initial items that are not in the range.
312 const auto& items
= blob_data_
->items();
313 for (current_item_index_
= 0;
314 current_item_index_
< items
.size() &&
315 offset
>= item_length_list_
[current_item_index_
];
316 ++current_item_index_
) {
317 offset
-= item_length_list_
[current_item_index_
];
320 // Set the offset that need to jump to for the first item in the range.
321 current_item_offset_
= offset
;
326 // Adjust the offset of the first stream if it is of file type.
327 const BlobDataItem
& item
= *items
.at(current_item_index_
);
328 if (IsFileType(item
.type())) {
329 DeleteCurrentFileReader();
330 CreateFileStreamReader(current_item_index_
, offset
);
334 bool BlobURLRequestJob::ReadItem() {
335 // Are we done with reading all the blob data?
336 if (remaining_bytes_
== 0)
339 const auto& items
= blob_data_
->items();
340 // If we get to the last item but still expect something to read, bail out
341 // since something is wrong.
342 if (current_item_index_
>= items
.size()) {
343 NotifyFailure(net::ERR_FAILED
);
347 // Compute the bytes to read for current item.
348 int bytes_to_read
= ComputeBytesToRead();
350 // If nothing to read for current item, advance to next item.
351 if (bytes_to_read
== 0) {
357 const BlobDataItem
& item
= *items
.at(current_item_index_
);
358 if (item
.type() == DataElement::TYPE_BYTES
)
359 return ReadBytesItem(item
, bytes_to_read
);
360 if (item
.type() == DataElement::TYPE_DISK_CACHE_ENTRY
)
361 return ReadDiskCacheEntryItem(item
, bytes_to_read
);
362 if (!IsFileType(item
.type())) {
366 storage::FileStreamReader
* const reader
=
367 GetFileStreamReader(current_item_index_
);
369 NotifyFailure(net::ERR_FAILED
);
373 return ReadFileItem(reader
, bytes_to_read
);
376 void BlobURLRequestJob::AdvanceItem() {
377 // Close the file if the current item is a file.
378 DeleteCurrentFileReader();
380 // Advance to the next item.
381 current_item_index_
++;
382 current_item_offset_
= 0;
385 void BlobURLRequestJob::AdvanceBytesRead(int result
) {
386 DCHECK_GT(result
, 0);
388 // Do we finish reading the current item?
389 current_item_offset_
+= result
;
390 if (current_item_offset_
== item_length_list_
[current_item_index_
])
393 // Subtract the remaining bytes.
394 remaining_bytes_
-= result
;
395 DCHECK_GE(remaining_bytes_
, 0);
397 // Adjust the read buffer.
398 read_buf_
->DidConsume(result
);
399 DCHECK_GE(read_buf_
->BytesRemaining(), 0);
402 bool BlobURLRequestJob::ReadBytesItem(const BlobDataItem
& item
,
404 TRACE_EVENT1("Blob", "BlobRequest::ReadBytesItem", "uuid",
406 DCHECK_GE(read_buf_
->BytesRemaining(), bytes_to_read
);
408 memcpy(read_buf_
->data(),
409 item
.bytes() + item
.offset() + current_item_offset_
,
412 AdvanceBytesRead(bytes_to_read
);
416 bool BlobURLRequestJob::ReadFileItem(FileStreamReader
* reader
,
418 DCHECK(!GetStatus().is_io_pending())
419 << "Can't begin IO while another IO operation is pending.";
420 DCHECK_GE(read_buf_
->BytesRemaining(), bytes_to_read
);
422 int chunk_number
= current_file_chunk_number_
++;
423 TRACE_EVENT_ASYNC_BEGIN1("Blob", "BlobRequest::ReadFileItem", this, "uuid",
426 reader
->Read(read_buf_
.get(), bytes_to_read
,
427 base::Bind(&BlobURLRequestJob::DidReadFile
,
428 weak_factory_
.GetWeakPtr(), chunk_number
));
430 AdvanceBytesRead(result
);
433 if (result
== net::ERR_IO_PENDING
)
434 SetStatus(net::URLRequestStatus(net::URLRequestStatus::IO_PENDING
, 0));
436 NotifyFailure(result
);
440 void BlobURLRequestJob::DidReadFile(int chunk_number
, int result
) {
441 DCHECK(GetStatus().is_io_pending())
442 << "Asynchronous IO completed while IO wasn't pending?";
443 TRACE_EVENT_ASYNC_END1("Blob", "BlobRequest::ReadFileItem", this, "uuid",
446 NotifyFailure(result
);
449 SetStatus(net::URLRequestStatus()); // Clear the IO_PENDING status
451 AdvanceBytesRead(result
);
453 // Otherwise, continue the reading.
455 if (ReadLoop(&bytes_read
))
456 NotifyReadComplete(bytes_read
);
459 void BlobURLRequestJob::DeleteCurrentFileReader() {
460 IndexToReaderMap::iterator found
= index_to_reader_
.find(current_item_index_
);
461 if (found
!= index_to_reader_
.end() && found
->second
) {
462 delete found
->second
;
463 index_to_reader_
.erase(found
);
467 bool BlobURLRequestJob::ReadDiskCacheEntryItem(const BlobDataItem
& item
,
469 DCHECK(!GetStatus().is_io_pending())
470 << "Can't begin IO while another IO operation is pending.";
471 DCHECK_GE(read_buf_
->BytesRemaining(), bytes_to_read
);
473 const int result
= item
.disk_cache_entry()->ReadData(
474 item
.disk_cache_stream_index(), current_item_offset_
, read_buf_
.get(),
475 bytes_to_read
, base::Bind(&BlobURLRequestJob::DidReadDiskCacheEntry
,
476 weak_factory_
.GetWeakPtr()));
478 AdvanceBytesRead(result
);
481 if (result
== net::ERR_IO_PENDING
)
482 SetStatus(net::URLRequestStatus(net::URLRequestStatus::IO_PENDING
, 0));
484 NotifyFailure(result
);
488 void BlobURLRequestJob::DidReadDiskCacheEntry(int result
) {
489 DCHECK(GetStatus().is_io_pending())
490 << "Asynchronous IO completed while IO wasn't pending?";
492 NotifyFailure(result
);
495 SetStatus(net::URLRequestStatus());
497 AdvanceBytesRead(result
);
500 if (ReadLoop(&bytes_read
))
501 NotifyReadComplete(bytes_read
);
504 int BlobURLRequestJob::BytesReadCompleted() {
505 int bytes_read
= read_buf_
->BytesConsumed();
510 int BlobURLRequestJob::ComputeBytesToRead() const {
511 int64 current_item_length
= item_length_list_
[current_item_index_
];
513 int64 item_remaining
= current_item_length
- current_item_offset_
;
514 int64 buf_remaining
= read_buf_
->BytesRemaining();
515 int64 max_remaining
= std::numeric_limits
<int>::max();
517 int64 min
= std::min(std::min(std::min(item_remaining
,
522 return static_cast<int>(min
);
525 bool BlobURLRequestJob::ReadLoop(int* bytes_read
) {
526 // Read until we encounter an error or could not get the data immediately.
527 while (remaining_bytes_
> 0 && read_buf_
->BytesRemaining() > 0) {
532 *bytes_read
= BytesReadCompleted();
536 void BlobURLRequestJob::NotifySuccess() {
537 net::HttpStatusCode status_code
= net::HTTP_OK
;
538 if (byte_range_set_
&& byte_range_
.IsValid())
539 status_code
= net::HTTP_PARTIAL_CONTENT
;
540 HeadersCompleted(status_code
);
543 void BlobURLRequestJob::NotifyFailure(int error_code
) {
546 // If we already return the headers on success, we can't change the headers
547 // now. Instead, we just error out.
548 if (response_info_
) {
549 NotifyDone(net::URLRequestStatus(net::URLRequestStatus::FAILED
,
554 net::HttpStatusCode status_code
= net::HTTP_INTERNAL_SERVER_ERROR
;
555 switch (error_code
) {
556 case net::ERR_ACCESS_DENIED
:
557 status_code
= net::HTTP_FORBIDDEN
;
559 case net::ERR_FILE_NOT_FOUND
:
560 status_code
= net::HTTP_NOT_FOUND
;
562 case net::ERR_METHOD_NOT_SUPPORTED
:
563 status_code
= net::HTTP_METHOD_NOT_ALLOWED
;
565 case net::ERR_REQUEST_RANGE_NOT_SATISFIABLE
:
566 status_code
= net::HTTP_REQUESTED_RANGE_NOT_SATISFIABLE
;
568 case net::ERR_FAILED
:
574 HeadersCompleted(status_code
);
577 void BlobURLRequestJob::HeadersCompleted(net::HttpStatusCode status_code
) {
578 std::string
status("HTTP/1.1 ");
579 status
.append(base::IntToString(status_code
));
581 status
.append(net::GetHttpReasonPhrase(status_code
));
582 status
.append("\0\0", 2);
583 net::HttpResponseHeaders
* headers
= new net::HttpResponseHeaders(status
);
585 if (status_code
== net::HTTP_OK
|| status_code
== net::HTTP_PARTIAL_CONTENT
) {
586 std::string
content_length_header(net::HttpRequestHeaders::kContentLength
);
587 content_length_header
.append(": ");
588 content_length_header
.append(base::Int64ToString(remaining_bytes_
));
589 headers
->AddHeader(content_length_header
);
590 if (status_code
== net::HTTP_PARTIAL_CONTENT
) {
591 DCHECK(byte_range_set_
);
592 DCHECK(byte_range_
.IsValid());
593 std::string
content_range_header(net::HttpResponseHeaders::kContentRange
);
594 content_range_header
.append(": bytes ");
595 content_range_header
.append(base::StringPrintf(
596 "%" PRId64
"-%" PRId64
,
597 byte_range_
.first_byte_position(), byte_range_
.last_byte_position()));
598 content_range_header
.append("/");
599 content_range_header
.append(base::StringPrintf("%" PRId64
, total_size_
));
600 headers
->AddHeader(content_range_header
);
602 if (!blob_data_
->content_type().empty()) {
603 std::string
content_type_header(net::HttpRequestHeaders::kContentType
);
604 content_type_header
.append(": ");
605 content_type_header
.append(blob_data_
->content_type());
606 headers
->AddHeader(content_type_header
);
608 if (!blob_data_
->content_disposition().empty()) {
609 std::string
content_disposition_header("Content-Disposition: ");
610 content_disposition_header
.append(blob_data_
->content_disposition());
611 headers
->AddHeader(content_disposition_header
);
615 response_info_
.reset(new net::HttpResponseInfo());
616 response_info_
->headers
= headers
;
618 set_expected_content_size(remaining_bytes_
);
620 NotifyHeadersComplete();
623 FileStreamReader
* BlobURLRequestJob::GetFileStreamReader(size_t index
) {
624 const auto& items
= blob_data_
->items();
625 DCHECK_LT(index
, items
.size());
626 const BlobDataItem
& item
= *items
.at(index
);
627 if (!IsFileType(item
.type()))
629 if (index_to_reader_
.find(index
) == index_to_reader_
.end()) {
630 if (!CreateFileStreamReader(index
, 0))
633 DCHECK(index_to_reader_
[index
]);
634 return index_to_reader_
[index
];
637 bool BlobURLRequestJob::CreateFileStreamReader(size_t index
,
638 int64 additional_offset
) {
639 const auto& items
= blob_data_
->items();
640 DCHECK_LT(index
, items
.size());
641 const BlobDataItem
& item
= *items
.at(index
);
642 DCHECK(IsFileType(item
.type()));
643 DCHECK_EQ(0U, index_to_reader_
.count(index
));
645 FileStreamReader
* reader
= nullptr;
646 switch (item
.type()) {
647 case DataElement::TYPE_FILE
:
648 reader
= FileStreamReader::CreateForLocalFile(
649 file_task_runner_
.get(), item
.path(),
650 item
.offset() + additional_offset
, item
.expected_modification_time());
652 index_to_reader_
[index
] = reader
;
655 case DataElement::TYPE_FILE_FILESYSTEM
:
656 reader
= file_system_context_
657 ->CreateFileStreamReader(
658 storage::FileSystemURL(file_system_context_
->CrackURL(
659 item
.filesystem_url())),
660 item
.offset() + additional_offset
,
661 item
.length() == std::numeric_limits
<uint64
>::max()
662 ? storage::kMaximumLength
663 : item
.length() - additional_offset
,
664 item
.expected_modification_time())
667 index_to_reader_
[index
] = reader
;
671 // The file stream reader may not be obtainable if the file is on an
672 // isolated file system, which has been unmounted.
683 } // namespace storage