1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "components/drive/file_cache.h"
10 #include "base/bind_helpers.h"
11 #include "base/callback_helpers.h"
12 #include "base/files/file_enumerator.h"
13 #include "base/files/file_util.h"
14 #include "base/location.h"
15 #include "base/logging.h"
16 #include "base/metrics/histogram.h"
17 #include "base/strings/string_util.h"
18 #include "base/strings/stringprintf.h"
19 #include "base/sys_info.h"
20 #include "components/drive/drive.pb.h"
21 #include "components/drive/drive_api_util.h"
22 #include "components/drive/file_system_core_util.h"
23 #include "components/drive/resource_metadata_storage.h"
24 #include "google_apis/drive/task_util.h"
25 #include "net/base/filename_util.h"
26 #include "net/base/mime_sniffer.h"
27 #include "net/base/mime_util.h"
33 // Returns ID extracted from the path.
34 std::string
GetIdFromPath(const base::FilePath
& path
) {
35 return util::UnescapeCacheFileName(path
.BaseName().AsUTF8Unsafe());
40 FileCache::FileCache(ResourceMetadataStorage
* storage
,
41 const base::FilePath
& cache_file_directory
,
42 base::SequencedTaskRunner
* blocking_task_runner
,
43 FreeDiskSpaceGetterInterface
* free_disk_space_getter
)
44 : cache_file_directory_(cache_file_directory
),
45 blocking_task_runner_(blocking_task_runner
),
47 free_disk_space_getter_(free_disk_space_getter
),
48 weak_ptr_factory_(this) {
49 DCHECK(blocking_task_runner_
.get());
52 FileCache::~FileCache() {
53 // Must be on the sequenced worker pool, as |metadata_| must be deleted on
54 // the sequenced worker pool.
55 AssertOnSequencedWorkerPool();
58 base::FilePath
FileCache::GetCacheFilePath(const std::string
& id
) const {
59 return cache_file_directory_
.Append(
60 base::FilePath::FromUTF8Unsafe(util::EscapeCacheFileName(id
)));
63 void FileCache::AssertOnSequencedWorkerPool() {
64 DCHECK(blocking_task_runner_
->RunsTasksOnCurrentThread());
67 bool FileCache::IsUnderFileCacheDirectory(const base::FilePath
& path
) const {
68 return cache_file_directory_
.IsParent(path
);
71 bool FileCache::FreeDiskSpaceIfNeededFor(int64 num_bytes
) {
72 AssertOnSequencedWorkerPool();
74 // Do nothing and return if we have enough space.
75 if (HasEnoughSpaceFor(num_bytes
, cache_file_directory_
))
78 // Otherwise, try to free up the disk space.
79 DVLOG(1) << "Freeing up disk space for " << num_bytes
;
81 // Remove all entries unless specially marked.
82 scoped_ptr
<ResourceMetadataStorage::Iterator
> it
= storage_
->GetIterator();
83 for (; !it
->IsAtEnd(); it
->Advance()) {
84 if (IsEvictable(it
->GetID(), it
->GetValue())) {
85 ResourceEntry
entry(it
->GetValue());
86 entry
.mutable_file_specific_info()->clear_cache_state();
87 storage_
->PutEntry(entry
);
93 // Remove all files which have no corresponding cache entries.
94 base::FileEnumerator
enumerator(cache_file_directory_
,
95 false, // not recursive
96 base::FileEnumerator::FILES
);
98 for (base::FilePath current
= enumerator
.Next(); !current
.empty();
99 current
= enumerator
.Next()) {
100 std::string id
= GetIdFromPath(current
);
101 FileError error
= storage_
->GetEntry(id
, &entry
);
102 if (error
== FILE_ERROR_NOT_FOUND
||
103 (error
== FILE_ERROR_OK
&&
104 !entry
.file_specific_info().cache_state().is_present()))
105 base::DeleteFile(current
, false /* recursive */);
106 else if (error
!= FILE_ERROR_OK
)
110 // Check the disk space again.
111 return HasEnoughSpaceFor(num_bytes
, cache_file_directory_
);
114 uint64_t FileCache::CalculateEvictableCacheSize() {
115 AssertOnSequencedWorkerPool();
117 uint64_t evictable_cache_size
= 0;
118 int64_t cache_size
= 0;
120 scoped_ptr
<ResourceMetadataStorage::Iterator
> it
= storage_
->GetIterator();
121 for (; !it
->IsAtEnd(); it
->Advance()) {
122 if (IsEvictable(it
->GetID(), it
->GetValue()) &&
123 base::GetFileSize(GetCacheFilePath(it
->GetID()), &cache_size
)) {
124 DCHECK_GE(cache_size
, 0);
125 evictable_cache_size
+= cache_size
;
132 return evictable_cache_size
;
135 FileError
FileCache::GetFile(const std::string
& id
,
136 base::FilePath
* cache_file_path
) {
137 AssertOnSequencedWorkerPool();
138 DCHECK(cache_file_path
);
141 FileError error
= storage_
->GetEntry(id
, &entry
);
142 if (error
!= FILE_ERROR_OK
)
144 if (!entry
.file_specific_info().cache_state().is_present())
145 return FILE_ERROR_NOT_FOUND
;
147 *cache_file_path
= GetCacheFilePath(id
);
148 return FILE_ERROR_OK
;
151 FileError
FileCache::Store(const std::string
& id
,
152 const std::string
& md5
,
153 const base::FilePath
& source_path
,
154 FileOperationType file_operation_type
) {
155 AssertOnSequencedWorkerPool();
158 FileError error
= storage_
->GetEntry(id
, &entry
);
159 if (error
!= FILE_ERROR_OK
)
163 if (file_operation_type
== FILE_OPERATION_COPY
) {
164 if (!base::GetFileSize(source_path
, &file_size
)) {
165 LOG(WARNING
) << "Couldn't get file size for: " << source_path
.value();
166 return FILE_ERROR_FAILED
;
169 if (!FreeDiskSpaceIfNeededFor(file_size
))
170 return FILE_ERROR_NO_LOCAL_SPACE
;
172 // If file is mounted, return error.
173 if (mounted_files_
.count(id
))
174 return FILE_ERROR_IN_USE
;
176 base::FilePath dest_path
= GetCacheFilePath(id
);
177 bool success
= false;
178 switch (file_operation_type
) {
179 case FILE_OPERATION_MOVE
:
180 success
= base::Move(source_path
, dest_path
);
182 case FILE_OPERATION_COPY
:
183 success
= base::CopyFile(source_path
, dest_path
);
190 LOG(ERROR
) << "Failed to store: "
191 << "source_path = " << source_path
.value() << ", "
192 << "dest_path = " << dest_path
.value() << ", "
193 << "file_operation_type = " << file_operation_type
;
194 return FILE_ERROR_FAILED
;
197 // Now that file operations have completed, update metadata.
198 FileCacheEntry
* cache_state
=
199 entry
.mutable_file_specific_info()->mutable_cache_state();
200 cache_state
->set_md5(md5
);
201 cache_state
->set_is_present(true);
203 cache_state
->set_is_dirty(true);
204 return storage_
->PutEntry(entry
);
207 FileError
FileCache::Pin(const std::string
& id
) {
208 AssertOnSequencedWorkerPool();
211 FileError error
= storage_
->GetEntry(id
, &entry
);
212 if (error
!= FILE_ERROR_OK
)
214 entry
.mutable_file_specific_info()->mutable_cache_state()->set_is_pinned(
216 return storage_
->PutEntry(entry
);
219 FileError
FileCache::Unpin(const std::string
& id
) {
220 AssertOnSequencedWorkerPool();
222 // Unpinning a file means its entry must exist in cache.
224 FileError error
= storage_
->GetEntry(id
, &entry
);
225 if (error
!= FILE_ERROR_OK
)
228 // Now that file operations have completed, update metadata.
229 if (entry
.file_specific_info().cache_state().is_present()) {
230 entry
.mutable_file_specific_info()->mutable_cache_state()->set_is_pinned(
233 // Remove the existing entry if we are unpinning a non-present file.
234 entry
.mutable_file_specific_info()->clear_cache_state();
236 error
= storage_
->PutEntry(entry
);
237 if (error
!= FILE_ERROR_OK
)
240 // Now it's a chance to free up space if needed.
241 FreeDiskSpaceIfNeededFor(0);
243 return FILE_ERROR_OK
;
246 FileError
FileCache::MarkAsMounted(const std::string
& id
,
247 base::FilePath
* cache_file_path
) {
248 AssertOnSequencedWorkerPool();
249 DCHECK(cache_file_path
);
251 // Get cache entry associated with the id and md5
253 FileError error
= storage_
->GetEntry(id
, &entry
);
254 if (error
!= FILE_ERROR_OK
)
256 if (!entry
.file_specific_info().cache_state().is_present())
257 return FILE_ERROR_NOT_FOUND
;
259 if (mounted_files_
.count(id
))
260 return FILE_ERROR_INVALID_OPERATION
;
262 base::FilePath path
= GetCacheFilePath(id
);
264 #if defined(OS_CHROMEOS)
265 // Ensure the file is readable to cros_disks. See crbug.com/236994.
266 if (!base::SetPosixFilePermissions(
268 base::FILE_PERMISSION_READ_BY_USER
|
269 base::FILE_PERMISSION_WRITE_BY_USER
|
270 base::FILE_PERMISSION_READ_BY_GROUP
|
271 base::FILE_PERMISSION_READ_BY_OTHERS
))
272 return FILE_ERROR_FAILED
;
275 mounted_files_
.insert(id
);
277 *cache_file_path
= path
;
278 return FILE_ERROR_OK
;
281 FileError
FileCache::OpenForWrite(
282 const std::string
& id
,
283 scoped_ptr
<base::ScopedClosureRunner
>* file_closer
) {
284 AssertOnSequencedWorkerPool();
286 // Marking a file dirty means its entry and actual file blob must exist in
289 FileError error
= storage_
->GetEntry(id
, &entry
);
290 if (error
!= FILE_ERROR_OK
)
292 if (!entry
.file_specific_info().cache_state().is_present()) {
293 LOG(WARNING
) << "Can't mark dirty a file that wasn't cached: " << id
;
294 return FILE_ERROR_NOT_FOUND
;
297 entry
.mutable_file_specific_info()->mutable_cache_state()->set_is_dirty(true);
298 entry
.mutable_file_specific_info()->mutable_cache_state()->clear_md5();
299 error
= storage_
->PutEntry(entry
);
300 if (error
!= FILE_ERROR_OK
)
303 write_opened_files_
[id
]++;
304 file_closer
->reset(new base::ScopedClosureRunner(
305 base::Bind(&google_apis::RunTaskWithTaskRunner
,
306 blocking_task_runner_
,
307 base::Bind(&FileCache::CloseForWrite
,
308 weak_ptr_factory_
.GetWeakPtr(),
310 return FILE_ERROR_OK
;
313 bool FileCache::IsOpenedForWrite(const std::string
& id
) {
314 AssertOnSequencedWorkerPool();
315 return write_opened_files_
.count(id
) != 0;
318 FileError
FileCache::UpdateMd5(const std::string
& id
) {
319 AssertOnSequencedWorkerPool();
321 if (IsOpenedForWrite(id
))
322 return FILE_ERROR_IN_USE
;
325 FileError error
= storage_
->GetEntry(id
, &entry
);
326 if (error
!= FILE_ERROR_OK
)
328 if (!entry
.file_specific_info().cache_state().is_present())
329 return FILE_ERROR_NOT_FOUND
;
331 const std::string
& md5
=
332 util::GetMd5Digest(GetCacheFilePath(id
), &in_shutdown_
);
333 if (in_shutdown_
.IsSet())
334 return FILE_ERROR_ABORT
;
336 return FILE_ERROR_NOT_FOUND
;
338 entry
.mutable_file_specific_info()->mutable_cache_state()->set_md5(md5
);
339 return storage_
->PutEntry(entry
);
342 FileError
FileCache::ClearDirty(const std::string
& id
) {
343 AssertOnSequencedWorkerPool();
345 if (IsOpenedForWrite(id
))
346 return FILE_ERROR_IN_USE
;
348 // Clearing a dirty file means its entry and actual file blob must exist in
351 FileError error
= storage_
->GetEntry(id
, &entry
);
352 if (error
!= FILE_ERROR_OK
)
354 if (!entry
.file_specific_info().cache_state().is_present()) {
355 LOG(WARNING
) << "Can't clear dirty state of a file that wasn't cached: "
357 return FILE_ERROR_NOT_FOUND
;
360 // If a file is not dirty (it should have been marked dirty via OpenForWrite),
361 // clearing its dirty state is an invalid operation.
362 if (!entry
.file_specific_info().cache_state().is_dirty()) {
363 LOG(WARNING
) << "Can't clear dirty state of a non-dirty file: " << id
;
364 return FILE_ERROR_INVALID_OPERATION
;
367 entry
.mutable_file_specific_info()->mutable_cache_state()->set_is_dirty(
369 return storage_
->PutEntry(entry
);
372 FileError
FileCache::Remove(const std::string
& id
) {
373 AssertOnSequencedWorkerPool();
377 // If entry doesn't exist, nothing to do.
378 FileError error
= storage_
->GetEntry(id
, &entry
);
379 if (error
== FILE_ERROR_NOT_FOUND
)
380 return FILE_ERROR_OK
;
381 if (error
!= FILE_ERROR_OK
)
383 if (!entry
.file_specific_info().has_cache_state())
384 return FILE_ERROR_OK
;
386 // Cannot delete a mounted file.
387 if (mounted_files_
.count(id
))
388 return FILE_ERROR_IN_USE
;
391 base::FilePath path
= GetCacheFilePath(id
);
392 if (!base::DeleteFile(path
, false /* recursive */))
393 return FILE_ERROR_FAILED
;
395 // Now that all file operations have completed, remove from metadata.
396 entry
.mutable_file_specific_info()->clear_cache_state();
397 return storage_
->PutEntry(entry
);
400 bool FileCache::ClearAll() {
401 AssertOnSequencedWorkerPool();
404 base::FileEnumerator
enumerator(cache_file_directory_
,
405 false, // not recursive
406 base::FileEnumerator::FILES
);
407 for (base::FilePath file
= enumerator
.Next(); !file
.empty();
408 file
= enumerator
.Next())
409 base::DeleteFile(file
, false /* recursive */);
414 bool FileCache::Initialize() {
415 AssertOnSequencedWorkerPool();
417 // Older versions do not clear MD5 when marking entries dirty.
418 // Clear MD5 of all dirty entries to deal with old data.
419 scoped_ptr
<ResourceMetadataStorage::Iterator
> it
= storage_
->GetIterator();
420 for (; !it
->IsAtEnd(); it
->Advance()) {
421 if (it
->GetValue().file_specific_info().cache_state().is_dirty()) {
422 ResourceEntry
new_entry(it
->GetValue());
423 new_entry
.mutable_file_specific_info()->mutable_cache_state()->
425 if (storage_
->PutEntry(new_entry
) != FILE_ERROR_OK
)
432 if (!RenameCacheFilesToNewFormat())
437 void FileCache::Destroy() {
438 DCHECK(thread_checker_
.CalledOnValidThread());
442 // Destroy myself on the blocking pool.
443 // Note that base::DeletePointer<> cannot be used as the destructor of this
445 blocking_task_runner_
->PostTask(
447 base::Bind(&FileCache::DestroyOnBlockingPool
, base::Unretained(this)));
450 void FileCache::DestroyOnBlockingPool() {
451 AssertOnSequencedWorkerPool();
455 bool FileCache::RecoverFilesFromCacheDirectory(
456 const base::FilePath
& dest_directory
,
457 const ResourceMetadataStorage::RecoveredCacheInfoMap
&
458 recovered_cache_info
) {
461 base::FileEnumerator
enumerator(cache_file_directory_
,
462 false, // not recursive
463 base::FileEnumerator::FILES
);
464 for (base::FilePath current
= enumerator
.Next(); !current
.empty();
465 current
= enumerator
.Next()) {
466 const std::string
& id
= GetIdFromPath(current
);
468 FileError error
= storage_
->GetEntry(id
, &entry
);
469 if (error
!= FILE_ERROR_OK
&& error
!= FILE_ERROR_NOT_FOUND
)
471 if (error
== FILE_ERROR_OK
&&
472 entry
.file_specific_info().cache_state().is_present()) {
473 // This file is managed by FileCache, no need to recover it.
477 // If a cache entry which is non-dirty and has matching MD5 is found in
478 // |recovered_cache_entries|, it means the current file is already uploaded
479 // to the server. Just delete it instead of recovering it.
480 ResourceMetadataStorage::RecoveredCacheInfoMap::const_iterator it
=
481 recovered_cache_info
.find(id
);
482 if (it
!= recovered_cache_info
.end()) {
483 // Due to the DB corruption, cache info might be recovered from old
484 // revision. Perform MD5 check even when is_dirty is false just in case.
485 if (!it
->second
.is_dirty
&&
486 it
->second
.md5
== util::GetMd5Digest(current
, &in_shutdown_
)) {
487 base::DeleteFile(current
, false /* recursive */);
492 // Read file contents to sniff mime type.
493 std::vector
<char> content(net::kMaxBytesToSniff
);
494 const int read_result
=
495 base::ReadFile(current
, &content
[0], content
.size());
496 if (read_result
< 0) {
497 LOG(WARNING
) << "Cannot read: " << current
.value();
500 if (read_result
== 0) // Skip empty files.
503 // Use recovered file name if available, otherwise decide file name with
504 // sniffed mime type.
505 base::FilePath
dest_base_name(FILE_PATH_LITERAL("file"));
506 std::string mime_type
;
507 if (it
!= recovered_cache_info
.end() && !it
->second
.title
.empty()) {
508 // We can use a file name recovered from the trashed DB.
509 dest_base_name
= base::FilePath::FromUTF8Unsafe(it
->second
.title
);
510 } else if (net::SniffMimeType(&content
[0], read_result
,
511 net::FilePathToFileURL(current
),
512 std::string(), &mime_type
) ||
513 net::SniffMimeTypeFromLocalData(&content
[0], read_result
,
515 // Change base name for common mime types.
516 if (net::MatchesMimeType("image/*", mime_type
)) {
517 dest_base_name
= base::FilePath(FILE_PATH_LITERAL("image"));
518 } else if (net::MatchesMimeType("video/*", mime_type
)) {
519 dest_base_name
= base::FilePath(FILE_PATH_LITERAL("video"));
520 } else if (net::MatchesMimeType("audio/*", mime_type
)) {
521 dest_base_name
= base::FilePath(FILE_PATH_LITERAL("audio"));
524 // Estimate extension from mime type.
525 std::vector
<base::FilePath::StringType
> extensions
;
526 base::FilePath::StringType extension
;
527 if (net::GetPreferredExtensionForMimeType(mime_type
, &extension
))
528 extensions
.push_back(extension
);
530 net::GetExtensionsForMimeType(mime_type
, &extensions
);
532 // Add extension if possible.
533 if (!extensions
.empty())
534 dest_base_name
= dest_base_name
.AddExtension(extensions
[0]);
537 // Add file number to the file name and move.
538 const base::FilePath
& dest_path
= dest_directory
.Append(dest_base_name
)
539 .InsertBeforeExtensionASCII(base::StringPrintf("%08d", file_number
++));
540 if (!base::CreateDirectory(dest_directory
) ||
541 !base::Move(current
, dest_path
)) {
542 LOG(WARNING
) << "Failed to move: " << current
.value()
543 << " to " << dest_path
.value();
547 UMA_HISTOGRAM_COUNTS("Drive.NumberOfCacheFilesRecoveredAfterDBCorruption",
552 FileError
FileCache::MarkAsUnmounted(const base::FilePath
& file_path
) {
553 AssertOnSequencedWorkerPool();
554 DCHECK(IsUnderFileCacheDirectory(file_path
));
556 std::string id
= GetIdFromPath(file_path
);
558 // Get the entry associated with the id.
560 FileError error
= storage_
->GetEntry(id
, &entry
);
561 if (error
!= FILE_ERROR_OK
)
564 std::set
<std::string
>::iterator it
= mounted_files_
.find(id
);
565 if (it
== mounted_files_
.end())
566 return FILE_ERROR_INVALID_OPERATION
;
568 mounted_files_
.erase(it
);
569 return FILE_ERROR_OK
;
572 bool FileCache::HasEnoughSpaceFor(int64 num_bytes
,
573 const base::FilePath
& path
) {
574 int64 free_space
= 0;
575 if (free_disk_space_getter_
)
576 free_space
= free_disk_space_getter_
->AmountOfFreeDiskSpace();
578 free_space
= base::SysInfo::AmountOfFreeDiskSpace(path
);
580 // Subtract this as if this portion does not exist.
581 free_space
-= drive::internal::kMinFreeSpaceInBytes
;
582 return (free_space
>= num_bytes
);
585 bool FileCache::RenameCacheFilesToNewFormat() {
586 base::FileEnumerator
enumerator(cache_file_directory_
,
587 false, // not recursive
588 base::FileEnumerator::FILES
);
589 for (base::FilePath current
= enumerator
.Next(); !current
.empty();
590 current
= enumerator
.Next()) {
591 base::FilePath new_path
= current
.RemoveExtension();
592 if (!new_path
.Extension().empty()) {
593 // Delete files with multiple extensions.
594 if (!base::DeleteFile(current
, false /* recursive */))
598 const std::string
& id
= GetIdFromPath(new_path
);
599 new_path
= GetCacheFilePath(util::CanonicalizeResourceId(id
));
600 if (new_path
!= current
&& !base::Move(current
, new_path
))
606 void FileCache::CloseForWrite(const std::string
& id
) {
607 AssertOnSequencedWorkerPool();
609 std::map
<std::string
, int>::iterator it
= write_opened_files_
.find(id
);
610 if (it
== write_opened_files_
.end())
613 DCHECK_LT(0, it
->second
);
616 write_opened_files_
.erase(it
);
618 // Update last modified date.
620 FileError error
= storage_
->GetEntry(id
, &entry
);
621 if (error
!= FILE_ERROR_OK
) {
622 LOG(ERROR
) << "Failed to get entry: " << id
<< ", "
623 << FileErrorToString(error
);
626 entry
.mutable_file_info()->set_last_modified(
627 base::Time::Now().ToInternalValue());
628 error
= storage_
->PutEntry(entry
);
629 if (error
!= FILE_ERROR_OK
) {
630 LOG(ERROR
) << "Failed to put entry: " << id
<< ", "
631 << FileErrorToString(error
);
635 bool FileCache::IsEvictable(const std::string
& id
, const ResourceEntry
& entry
) {
636 return entry
.file_specific_info().has_cache_state() &&
637 !entry
.file_specific_info().cache_state().is_pinned() &&
638 !entry
.file_specific_info().cache_state().is_dirty() &&
639 !mounted_files_
.count(id
);
642 } // namespace internal