Add 'did_proceed' and 'repeat_visit' to ClientMalwareReportRequest to track CTR.
[chromium-blink-merge.git] / components / drive / file_cache.cc
blob572316c6bcbc83575ae68eea1025d770d2d133c3
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "components/drive/file_cache.h"
7 #include <vector>
9 #include "base/bind.h"
10 #include "base/bind_helpers.h"
11 #include "base/callback_helpers.h"
12 #include "base/files/file_enumerator.h"
13 #include "base/files/file_util.h"
14 #include "base/location.h"
15 #include "base/logging.h"
16 #include "base/metrics/histogram.h"
17 #include "base/strings/string_util.h"
18 #include "base/strings/stringprintf.h"
19 #include "base/sys_info.h"
20 #include "components/drive/drive.pb.h"
21 #include "components/drive/drive_api_util.h"
22 #include "components/drive/file_system_core_util.h"
23 #include "components/drive/resource_metadata_storage.h"
24 #include "google_apis/drive/task_util.h"
25 #include "net/base/filename_util.h"
26 #include "net/base/mime_sniffer.h"
27 #include "net/base/mime_util.h"
29 namespace drive {
30 namespace internal {
31 namespace {
33 // Returns ID extracted from the path.
34 std::string GetIdFromPath(const base::FilePath& path) {
35 return util::UnescapeCacheFileName(path.BaseName().AsUTF8Unsafe());
38 } // namespace
40 FileCache::FileCache(ResourceMetadataStorage* storage,
41 const base::FilePath& cache_file_directory,
42 base::SequencedTaskRunner* blocking_task_runner,
43 FreeDiskSpaceGetterInterface* free_disk_space_getter)
44 : cache_file_directory_(cache_file_directory),
45 blocking_task_runner_(blocking_task_runner),
46 storage_(storage),
47 free_disk_space_getter_(free_disk_space_getter),
48 weak_ptr_factory_(this) {
49 DCHECK(blocking_task_runner_.get());
52 FileCache::~FileCache() {
53 // Must be on the sequenced worker pool, as |metadata_| must be deleted on
54 // the sequenced worker pool.
55 AssertOnSequencedWorkerPool();
58 base::FilePath FileCache::GetCacheFilePath(const std::string& id) const {
59 return cache_file_directory_.Append(
60 base::FilePath::FromUTF8Unsafe(util::EscapeCacheFileName(id)));
63 void FileCache::AssertOnSequencedWorkerPool() {
64 DCHECK(blocking_task_runner_->RunsTasksOnCurrentThread());
67 bool FileCache::IsUnderFileCacheDirectory(const base::FilePath& path) const {
68 return cache_file_directory_.IsParent(path);
71 bool FileCache::FreeDiskSpaceIfNeededFor(int64 num_bytes) {
72 AssertOnSequencedWorkerPool();
74 // Do nothing and return if we have enough space.
75 if (HasEnoughSpaceFor(num_bytes, cache_file_directory_))
76 return true;
78 // Otherwise, try to free up the disk space.
79 DVLOG(1) << "Freeing up disk space for " << num_bytes;
81 // Remove all entries unless specially marked.
82 scoped_ptr<ResourceMetadataStorage::Iterator> it = storage_->GetIterator();
83 for (; !it->IsAtEnd(); it->Advance()) {
84 if (IsEvictable(it->GetID(), it->GetValue())) {
85 ResourceEntry entry(it->GetValue());
86 entry.mutable_file_specific_info()->clear_cache_state();
87 storage_->PutEntry(entry);
90 if (it->HasError())
91 return false;
93 // Remove all files which have no corresponding cache entries.
94 base::FileEnumerator enumerator(cache_file_directory_,
95 false, // not recursive
96 base::FileEnumerator::FILES);
97 ResourceEntry entry;
98 for (base::FilePath current = enumerator.Next(); !current.empty();
99 current = enumerator.Next()) {
100 std::string id = GetIdFromPath(current);
101 FileError error = storage_->GetEntry(id, &entry);
102 if (error == FILE_ERROR_NOT_FOUND ||
103 (error == FILE_ERROR_OK &&
104 !entry.file_specific_info().cache_state().is_present()))
105 base::DeleteFile(current, false /* recursive */);
106 else if (error != FILE_ERROR_OK)
107 return false;
110 // Check the disk space again.
111 return HasEnoughSpaceFor(num_bytes, cache_file_directory_);
114 uint64_t FileCache::CalculateEvictableCacheSize() {
115 AssertOnSequencedWorkerPool();
117 uint64_t evictable_cache_size = 0;
118 int64_t cache_size = 0;
120 scoped_ptr<ResourceMetadataStorage::Iterator> it = storage_->GetIterator();
121 for (; !it->IsAtEnd(); it->Advance()) {
122 if (IsEvictable(it->GetID(), it->GetValue()) &&
123 base::GetFileSize(GetCacheFilePath(it->GetID()), &cache_size)) {
124 DCHECK_GE(cache_size, 0);
125 evictable_cache_size += cache_size;
129 if (it->HasError())
130 return 0;
132 return evictable_cache_size;
135 FileError FileCache::GetFile(const std::string& id,
136 base::FilePath* cache_file_path) {
137 AssertOnSequencedWorkerPool();
138 DCHECK(cache_file_path);
140 ResourceEntry entry;
141 FileError error = storage_->GetEntry(id, &entry);
142 if (error != FILE_ERROR_OK)
143 return error;
144 if (!entry.file_specific_info().cache_state().is_present())
145 return FILE_ERROR_NOT_FOUND;
147 *cache_file_path = GetCacheFilePath(id);
148 return FILE_ERROR_OK;
151 FileError FileCache::Store(const std::string& id,
152 const std::string& md5,
153 const base::FilePath& source_path,
154 FileOperationType file_operation_type) {
155 AssertOnSequencedWorkerPool();
157 ResourceEntry entry;
158 FileError error = storage_->GetEntry(id, &entry);
159 if (error != FILE_ERROR_OK)
160 return error;
162 int64 file_size = 0;
163 if (file_operation_type == FILE_OPERATION_COPY) {
164 if (!base::GetFileSize(source_path, &file_size)) {
165 LOG(WARNING) << "Couldn't get file size for: " << source_path.value();
166 return FILE_ERROR_FAILED;
169 if (!FreeDiskSpaceIfNeededFor(file_size))
170 return FILE_ERROR_NO_LOCAL_SPACE;
172 // If file is mounted, return error.
173 if (mounted_files_.count(id))
174 return FILE_ERROR_IN_USE;
176 base::FilePath dest_path = GetCacheFilePath(id);
177 bool success = false;
178 switch (file_operation_type) {
179 case FILE_OPERATION_MOVE:
180 success = base::Move(source_path, dest_path);
181 break;
182 case FILE_OPERATION_COPY:
183 success = base::CopyFile(source_path, dest_path);
184 break;
185 default:
186 NOTREACHED();
189 if (!success) {
190 LOG(ERROR) << "Failed to store: "
191 << "source_path = " << source_path.value() << ", "
192 << "dest_path = " << dest_path.value() << ", "
193 << "file_operation_type = " << file_operation_type;
194 return FILE_ERROR_FAILED;
197 // Now that file operations have completed, update metadata.
198 FileCacheEntry* cache_state =
199 entry.mutable_file_specific_info()->mutable_cache_state();
200 cache_state->set_md5(md5);
201 cache_state->set_is_present(true);
202 if (md5.empty())
203 cache_state->set_is_dirty(true);
204 return storage_->PutEntry(entry);
207 FileError FileCache::Pin(const std::string& id) {
208 AssertOnSequencedWorkerPool();
210 ResourceEntry entry;
211 FileError error = storage_->GetEntry(id, &entry);
212 if (error != FILE_ERROR_OK)
213 return error;
214 entry.mutable_file_specific_info()->mutable_cache_state()->set_is_pinned(
215 true);
216 return storage_->PutEntry(entry);
219 FileError FileCache::Unpin(const std::string& id) {
220 AssertOnSequencedWorkerPool();
222 // Unpinning a file means its entry must exist in cache.
223 ResourceEntry entry;
224 FileError error = storage_->GetEntry(id, &entry);
225 if (error != FILE_ERROR_OK)
226 return error;
228 // Now that file operations have completed, update metadata.
229 if (entry.file_specific_info().cache_state().is_present()) {
230 entry.mutable_file_specific_info()->mutable_cache_state()->set_is_pinned(
231 false);
232 } else {
233 // Remove the existing entry if we are unpinning a non-present file.
234 entry.mutable_file_specific_info()->clear_cache_state();
236 error = storage_->PutEntry(entry);
237 if (error != FILE_ERROR_OK)
238 return error;
240 // Now it's a chance to free up space if needed.
241 FreeDiskSpaceIfNeededFor(0);
243 return FILE_ERROR_OK;
246 FileError FileCache::MarkAsMounted(const std::string& id,
247 base::FilePath* cache_file_path) {
248 AssertOnSequencedWorkerPool();
249 DCHECK(cache_file_path);
251 // Get cache entry associated with the id and md5
252 ResourceEntry entry;
253 FileError error = storage_->GetEntry(id, &entry);
254 if (error != FILE_ERROR_OK)
255 return error;
256 if (!entry.file_specific_info().cache_state().is_present())
257 return FILE_ERROR_NOT_FOUND;
259 if (mounted_files_.count(id))
260 return FILE_ERROR_INVALID_OPERATION;
262 base::FilePath path = GetCacheFilePath(id);
264 #if defined(OS_CHROMEOS)
265 // Ensure the file is readable to cros_disks. See crbug.com/236994.
266 if (!base::SetPosixFilePermissions(
267 path,
268 base::FILE_PERMISSION_READ_BY_USER |
269 base::FILE_PERMISSION_WRITE_BY_USER |
270 base::FILE_PERMISSION_READ_BY_GROUP |
271 base::FILE_PERMISSION_READ_BY_OTHERS))
272 return FILE_ERROR_FAILED;
273 #endif
275 mounted_files_.insert(id);
277 *cache_file_path = path;
278 return FILE_ERROR_OK;
281 FileError FileCache::OpenForWrite(
282 const std::string& id,
283 scoped_ptr<base::ScopedClosureRunner>* file_closer) {
284 AssertOnSequencedWorkerPool();
286 // Marking a file dirty means its entry and actual file blob must exist in
287 // cache.
288 ResourceEntry entry;
289 FileError error = storage_->GetEntry(id, &entry);
290 if (error != FILE_ERROR_OK)
291 return error;
292 if (!entry.file_specific_info().cache_state().is_present()) {
293 LOG(WARNING) << "Can't mark dirty a file that wasn't cached: " << id;
294 return FILE_ERROR_NOT_FOUND;
297 entry.mutable_file_specific_info()->mutable_cache_state()->set_is_dirty(true);
298 entry.mutable_file_specific_info()->mutable_cache_state()->clear_md5();
299 error = storage_->PutEntry(entry);
300 if (error != FILE_ERROR_OK)
301 return error;
303 write_opened_files_[id]++;
304 file_closer->reset(new base::ScopedClosureRunner(
305 base::Bind(&google_apis::RunTaskWithTaskRunner,
306 blocking_task_runner_,
307 base::Bind(&FileCache::CloseForWrite,
308 weak_ptr_factory_.GetWeakPtr(),
309 id))));
310 return FILE_ERROR_OK;
313 bool FileCache::IsOpenedForWrite(const std::string& id) {
314 AssertOnSequencedWorkerPool();
315 return write_opened_files_.count(id) != 0;
318 FileError FileCache::UpdateMd5(const std::string& id) {
319 AssertOnSequencedWorkerPool();
321 if (IsOpenedForWrite(id))
322 return FILE_ERROR_IN_USE;
324 ResourceEntry entry;
325 FileError error = storage_->GetEntry(id, &entry);
326 if (error != FILE_ERROR_OK)
327 return error;
328 if (!entry.file_specific_info().cache_state().is_present())
329 return FILE_ERROR_NOT_FOUND;
331 const std::string& md5 =
332 util::GetMd5Digest(GetCacheFilePath(id), &in_shutdown_);
333 if (in_shutdown_.IsSet())
334 return FILE_ERROR_ABORT;
335 if (md5.empty())
336 return FILE_ERROR_NOT_FOUND;
338 entry.mutable_file_specific_info()->mutable_cache_state()->set_md5(md5);
339 return storage_->PutEntry(entry);
342 FileError FileCache::ClearDirty(const std::string& id) {
343 AssertOnSequencedWorkerPool();
345 if (IsOpenedForWrite(id))
346 return FILE_ERROR_IN_USE;
348 // Clearing a dirty file means its entry and actual file blob must exist in
349 // cache.
350 ResourceEntry entry;
351 FileError error = storage_->GetEntry(id, &entry);
352 if (error != FILE_ERROR_OK)
353 return error;
354 if (!entry.file_specific_info().cache_state().is_present()) {
355 LOG(WARNING) << "Can't clear dirty state of a file that wasn't cached: "
356 << id;
357 return FILE_ERROR_NOT_FOUND;
360 // If a file is not dirty (it should have been marked dirty via OpenForWrite),
361 // clearing its dirty state is an invalid operation.
362 if (!entry.file_specific_info().cache_state().is_dirty()) {
363 LOG(WARNING) << "Can't clear dirty state of a non-dirty file: " << id;
364 return FILE_ERROR_INVALID_OPERATION;
367 entry.mutable_file_specific_info()->mutable_cache_state()->set_is_dirty(
368 false);
369 return storage_->PutEntry(entry);
372 FileError FileCache::Remove(const std::string& id) {
373 AssertOnSequencedWorkerPool();
375 ResourceEntry entry;
377 // If entry doesn't exist, nothing to do.
378 FileError error = storage_->GetEntry(id, &entry);
379 if (error == FILE_ERROR_NOT_FOUND)
380 return FILE_ERROR_OK;
381 if (error != FILE_ERROR_OK)
382 return error;
383 if (!entry.file_specific_info().has_cache_state())
384 return FILE_ERROR_OK;
386 // Cannot delete a mounted file.
387 if (mounted_files_.count(id))
388 return FILE_ERROR_IN_USE;
390 // Delete the file.
391 base::FilePath path = GetCacheFilePath(id);
392 if (!base::DeleteFile(path, false /* recursive */))
393 return FILE_ERROR_FAILED;
395 // Now that all file operations have completed, remove from metadata.
396 entry.mutable_file_specific_info()->clear_cache_state();
397 return storage_->PutEntry(entry);
400 bool FileCache::ClearAll() {
401 AssertOnSequencedWorkerPool();
403 // Remove files.
404 base::FileEnumerator enumerator(cache_file_directory_,
405 false, // not recursive
406 base::FileEnumerator::FILES);
407 for (base::FilePath file = enumerator.Next(); !file.empty();
408 file = enumerator.Next())
409 base::DeleteFile(file, false /* recursive */);
411 return true;
414 bool FileCache::Initialize() {
415 AssertOnSequencedWorkerPool();
417 // Older versions do not clear MD5 when marking entries dirty.
418 // Clear MD5 of all dirty entries to deal with old data.
419 scoped_ptr<ResourceMetadataStorage::Iterator> it = storage_->GetIterator();
420 for (; !it->IsAtEnd(); it->Advance()) {
421 if (it->GetValue().file_specific_info().cache_state().is_dirty()) {
422 ResourceEntry new_entry(it->GetValue());
423 new_entry.mutable_file_specific_info()->mutable_cache_state()->
424 clear_md5();
425 if (storage_->PutEntry(new_entry) != FILE_ERROR_OK)
426 return false;
429 if (it->HasError())
430 return false;
432 if (!RenameCacheFilesToNewFormat())
433 return false;
434 return true;
437 void FileCache::Destroy() {
438 DCHECK(thread_checker_.CalledOnValidThread());
440 in_shutdown_.Set();
442 // Destroy myself on the blocking pool.
443 // Note that base::DeletePointer<> cannot be used as the destructor of this
444 // class is private.
445 blocking_task_runner_->PostTask(
446 FROM_HERE,
447 base::Bind(&FileCache::DestroyOnBlockingPool, base::Unretained(this)));
450 void FileCache::DestroyOnBlockingPool() {
451 AssertOnSequencedWorkerPool();
452 delete this;
455 bool FileCache::RecoverFilesFromCacheDirectory(
456 const base::FilePath& dest_directory,
457 const ResourceMetadataStorage::RecoveredCacheInfoMap&
458 recovered_cache_info) {
459 int file_number = 1;
461 base::FileEnumerator enumerator(cache_file_directory_,
462 false, // not recursive
463 base::FileEnumerator::FILES);
464 for (base::FilePath current = enumerator.Next(); !current.empty();
465 current = enumerator.Next()) {
466 const std::string& id = GetIdFromPath(current);
467 ResourceEntry entry;
468 FileError error = storage_->GetEntry(id, &entry);
469 if (error != FILE_ERROR_OK && error != FILE_ERROR_NOT_FOUND)
470 return false;
471 if (error == FILE_ERROR_OK &&
472 entry.file_specific_info().cache_state().is_present()) {
473 // This file is managed by FileCache, no need to recover it.
474 continue;
477 // If a cache entry which is non-dirty and has matching MD5 is found in
478 // |recovered_cache_entries|, it means the current file is already uploaded
479 // to the server. Just delete it instead of recovering it.
480 ResourceMetadataStorage::RecoveredCacheInfoMap::const_iterator it =
481 recovered_cache_info.find(id);
482 if (it != recovered_cache_info.end()) {
483 // Due to the DB corruption, cache info might be recovered from old
484 // revision. Perform MD5 check even when is_dirty is false just in case.
485 if (!it->second.is_dirty &&
486 it->second.md5 == util::GetMd5Digest(current, &in_shutdown_)) {
487 base::DeleteFile(current, false /* recursive */);
488 continue;
492 // Read file contents to sniff mime type.
493 std::vector<char> content(net::kMaxBytesToSniff);
494 const int read_result =
495 base::ReadFile(current, &content[0], content.size());
496 if (read_result < 0) {
497 LOG(WARNING) << "Cannot read: " << current.value();
498 return false;
500 if (read_result == 0) // Skip empty files.
501 continue;
503 // Use recovered file name if available, otherwise decide file name with
504 // sniffed mime type.
505 base::FilePath dest_base_name(FILE_PATH_LITERAL("file"));
506 std::string mime_type;
507 if (it != recovered_cache_info.end() && !it->second.title.empty()) {
508 // We can use a file name recovered from the trashed DB.
509 dest_base_name = base::FilePath::FromUTF8Unsafe(it->second.title);
510 } else if (net::SniffMimeType(&content[0], read_result,
511 net::FilePathToFileURL(current),
512 std::string(), &mime_type) ||
513 net::SniffMimeTypeFromLocalData(&content[0], read_result,
514 &mime_type)) {
515 // Change base name for common mime types.
516 if (net::MatchesMimeType("image/*", mime_type)) {
517 dest_base_name = base::FilePath(FILE_PATH_LITERAL("image"));
518 } else if (net::MatchesMimeType("video/*", mime_type)) {
519 dest_base_name = base::FilePath(FILE_PATH_LITERAL("video"));
520 } else if (net::MatchesMimeType("audio/*", mime_type)) {
521 dest_base_name = base::FilePath(FILE_PATH_LITERAL("audio"));
524 // Estimate extension from mime type.
525 std::vector<base::FilePath::StringType> extensions;
526 base::FilePath::StringType extension;
527 if (net::GetPreferredExtensionForMimeType(mime_type, &extension))
528 extensions.push_back(extension);
529 else
530 net::GetExtensionsForMimeType(mime_type, &extensions);
532 // Add extension if possible.
533 if (!extensions.empty())
534 dest_base_name = dest_base_name.AddExtension(extensions[0]);
537 // Add file number to the file name and move.
538 const base::FilePath& dest_path = dest_directory.Append(dest_base_name)
539 .InsertBeforeExtensionASCII(base::StringPrintf("%08d", file_number++));
540 if (!base::CreateDirectory(dest_directory) ||
541 !base::Move(current, dest_path)) {
542 LOG(WARNING) << "Failed to move: " << current.value()
543 << " to " << dest_path.value();
544 return false;
547 UMA_HISTOGRAM_COUNTS("Drive.NumberOfCacheFilesRecoveredAfterDBCorruption",
548 file_number - 1);
549 return true;
552 FileError FileCache::MarkAsUnmounted(const base::FilePath& file_path) {
553 AssertOnSequencedWorkerPool();
554 DCHECK(IsUnderFileCacheDirectory(file_path));
556 std::string id = GetIdFromPath(file_path);
558 // Get the entry associated with the id.
559 ResourceEntry entry;
560 FileError error = storage_->GetEntry(id, &entry);
561 if (error != FILE_ERROR_OK)
562 return error;
564 std::set<std::string>::iterator it = mounted_files_.find(id);
565 if (it == mounted_files_.end())
566 return FILE_ERROR_INVALID_OPERATION;
568 mounted_files_.erase(it);
569 return FILE_ERROR_OK;
572 bool FileCache::HasEnoughSpaceFor(int64 num_bytes,
573 const base::FilePath& path) {
574 int64 free_space = 0;
575 if (free_disk_space_getter_)
576 free_space = free_disk_space_getter_->AmountOfFreeDiskSpace();
577 else
578 free_space = base::SysInfo::AmountOfFreeDiskSpace(path);
580 // Subtract this as if this portion does not exist.
581 free_space -= drive::internal::kMinFreeSpaceInBytes;
582 return (free_space >= num_bytes);
585 bool FileCache::RenameCacheFilesToNewFormat() {
586 base::FileEnumerator enumerator(cache_file_directory_,
587 false, // not recursive
588 base::FileEnumerator::FILES);
589 for (base::FilePath current = enumerator.Next(); !current.empty();
590 current = enumerator.Next()) {
591 base::FilePath new_path = current.RemoveExtension();
592 if (!new_path.Extension().empty()) {
593 // Delete files with multiple extensions.
594 if (!base::DeleteFile(current, false /* recursive */))
595 return false;
596 continue;
598 const std::string& id = GetIdFromPath(new_path);
599 new_path = GetCacheFilePath(util::CanonicalizeResourceId(id));
600 if (new_path != current && !base::Move(current, new_path))
601 return false;
603 return true;
606 void FileCache::CloseForWrite(const std::string& id) {
607 AssertOnSequencedWorkerPool();
609 std::map<std::string, int>::iterator it = write_opened_files_.find(id);
610 if (it == write_opened_files_.end())
611 return;
613 DCHECK_LT(0, it->second);
614 --it->second;
615 if (it->second == 0)
616 write_opened_files_.erase(it);
618 // Update last modified date.
619 ResourceEntry entry;
620 FileError error = storage_->GetEntry(id, &entry);
621 if (error != FILE_ERROR_OK) {
622 LOG(ERROR) << "Failed to get entry: " << id << ", "
623 << FileErrorToString(error);
624 return;
626 entry.mutable_file_info()->set_last_modified(
627 base::Time::Now().ToInternalValue());
628 error = storage_->PutEntry(entry);
629 if (error != FILE_ERROR_OK) {
630 LOG(ERROR) << "Failed to put entry: " << id << ", "
631 << FileErrorToString(error);
635 bool FileCache::IsEvictable(const std::string& id, const ResourceEntry& entry) {
636 return entry.file_specific_info().has_cache_state() &&
637 !entry.file_specific_info().cache_state().is_pinned() &&
638 !entry.file_specific_info().cache_state().is_dirty() &&
639 !mounted_files_.count(id);
642 } // namespace internal
643 } // namespace drive