1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/sync_file_system/local/local_file_change_tracker.h"
9 #include "base/location.h"
10 #include "base/logging.h"
11 #include "base/sequenced_task_runner.h"
12 #include "base/stl_util.h"
13 #include "chrome/browser/sync_file_system/local/local_file_sync_status.h"
14 #include "chrome/browser/sync_file_system/syncable_file_system_util.h"
15 #include "storage/browser/fileapi/file_system_context.h"
16 #include "storage/browser/fileapi/file_system_file_util.h"
17 #include "storage/browser/fileapi/file_system_operation_context.h"
18 #include "storage/common/fileapi/file_system_util.h"
19 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
20 #include "third_party/leveldatabase/src/include/leveldb/db.h"
21 #include "third_party/leveldatabase/src/include/leveldb/env.h"
22 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
24 using storage::FileSystemContext
;
25 using storage::FileSystemFileUtil
;
26 using storage::FileSystemOperationContext
;
27 using storage::FileSystemURL
;
28 using storage::FileSystemURLSet
;
30 namespace sync_file_system
{
33 const base::FilePath::CharType kDatabaseName
[] =
34 FILE_PATH_LITERAL("LocalFileChangeTracker");
35 const char kMark
[] = "d";
38 // A database class that stores local file changes in a local database. This
39 // object must be destructed on file_task_runner.
40 class LocalFileChangeTracker::TrackerDB
{
42 TrackerDB(const base::FilePath
& base_path
,
43 leveldb::Env
* env_override
);
45 SyncStatusCode
MarkDirty(const std::string
& url
);
46 SyncStatusCode
ClearDirty(const std::string
& url
);
47 SyncStatusCode
GetDirtyEntries(
48 std::queue
<FileSystemURL
>* dirty_files
);
49 SyncStatusCode
WriteBatch(scoped_ptr
<leveldb::WriteBatch
> batch
);
57 SyncStatusCode
Init(RecoveryOption recovery_option
);
58 SyncStatusCode
Repair(const std::string
& db_path
);
59 void HandleError(const tracked_objects::Location
& from_here
,
60 const leveldb::Status
& status
);
62 const base::FilePath base_path_
;
63 leveldb::Env
* env_override_
;
64 scoped_ptr
<leveldb::DB
> db_
;
65 SyncStatusCode db_status_
;
67 DISALLOW_COPY_AND_ASSIGN(TrackerDB
);
70 LocalFileChangeTracker::ChangeInfo::ChangeInfo() : change_seq(-1) {}
71 LocalFileChangeTracker::ChangeInfo::~ChangeInfo() {}
73 // LocalFileChangeTracker ------------------------------------------------------
75 LocalFileChangeTracker::LocalFileChangeTracker(
76 const base::FilePath
& base_path
,
77 leveldb::Env
* env_override
,
78 base::SequencedTaskRunner
* file_task_runner
)
79 : initialized_(false),
80 file_task_runner_(file_task_runner
),
81 tracker_db_(new TrackerDB(base_path
, env_override
)),
82 current_change_seq_number_(0),
86 LocalFileChangeTracker::~LocalFileChangeTracker() {
87 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
91 void LocalFileChangeTracker::OnStartUpdate(const FileSystemURL
& url
) {
92 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
93 if (ContainsKey(changes_
, url
) || ContainsKey(demoted_changes_
, url
))
95 // TODO(nhiroki): propagate the error code (see http://crbug.com/152127).
96 MarkDirtyOnDatabase(url
);
99 void LocalFileChangeTracker::OnEndUpdate(const FileSystemURL
& url
) {}
101 void LocalFileChangeTracker::OnCreateFile(const FileSystemURL
& url
) {
102 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
103 SYNC_FILE_TYPE_FILE
));
106 void LocalFileChangeTracker::OnCreateFileFrom(const FileSystemURL
& url
,
107 const FileSystemURL
& src
) {
108 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
109 SYNC_FILE_TYPE_FILE
));
112 void LocalFileChangeTracker::OnRemoveFile(const FileSystemURL
& url
) {
113 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_DELETE
,
114 SYNC_FILE_TYPE_FILE
));
117 void LocalFileChangeTracker::OnModifyFile(const FileSystemURL
& url
) {
118 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
119 SYNC_FILE_TYPE_FILE
));
122 void LocalFileChangeTracker::OnCreateDirectory(const FileSystemURL
& url
) {
123 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
124 SYNC_FILE_TYPE_DIRECTORY
));
127 void LocalFileChangeTracker::OnRemoveDirectory(const FileSystemURL
& url
) {
128 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_DELETE
,
129 SYNC_FILE_TYPE_DIRECTORY
));
132 void LocalFileChangeTracker::GetNextChangedURLs(
133 std::deque
<FileSystemURL
>* urls
, int max_urls
) {
134 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
137 // Mildly prioritizes the URLs that older changes and have not been updated
139 for (ChangeSeqMap::iterator iter
= change_seqs_
.begin();
140 iter
!= change_seqs_
.end() &&
141 (max_urls
== 0 || urls
->size() < static_cast<size_t>(max_urls
));
143 urls
->push_back(iter
->second
);
147 void LocalFileChangeTracker::GetChangesForURL(
148 const FileSystemURL
& url
, FileChangeList
* changes
) {
149 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
152 FileChangeMap::iterator found
= changes_
.find(url
);
153 if (found
== changes_
.end()) {
154 found
= demoted_changes_
.find(url
);
155 if (found
== demoted_changes_
.end())
158 *changes
= found
->second
.change_list
;
161 void LocalFileChangeTracker::ClearChangesForURL(const FileSystemURL
& url
) {
162 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
163 ClearDirtyOnDatabase(url
);
164 mirror_changes_
.erase(url
);
165 demoted_changes_
.erase(url
);
166 FileChangeMap::iterator found
= changes_
.find(url
);
167 if (found
== changes_
.end())
169 change_seqs_
.erase(found
->second
.change_seq
);
170 changes_
.erase(found
);
174 void LocalFileChangeTracker::CreateFreshMirrorForURL(
175 const storage::FileSystemURL
& url
) {
176 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
177 DCHECK(!ContainsKey(mirror_changes_
, url
));
178 mirror_changes_
[url
] = ChangeInfo();
181 void LocalFileChangeTracker::RemoveMirrorAndCommitChangesForURL(
182 const storage::FileSystemURL
& url
) {
183 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
184 FileChangeMap::iterator found
= mirror_changes_
.find(url
);
185 if (found
== mirror_changes_
.end())
187 mirror_changes_
.erase(found
);
189 if (ContainsKey(changes_
, url
) || ContainsKey(demoted_changes_
, url
))
190 MarkDirtyOnDatabase(url
);
192 ClearDirtyOnDatabase(url
);
196 void LocalFileChangeTracker::ResetToMirrorAndCommitChangesForURL(
197 const storage::FileSystemURL
& url
) {
198 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
199 FileChangeMap::iterator found
= mirror_changes_
.find(url
);
200 if (found
== mirror_changes_
.end() || found
->second
.change_list
.empty()) {
201 ClearChangesForURL(url
);
204 const ChangeInfo
& info
= found
->second
;
205 if (ContainsKey(demoted_changes_
, url
)) {
206 DCHECK(!ContainsKey(changes_
, url
));
207 demoted_changes_
[url
] = info
;
209 DCHECK(!ContainsKey(demoted_changes_
, url
));
210 change_seqs_
[info
.change_seq
] = url
;
211 changes_
[url
] = info
;
213 RemoveMirrorAndCommitChangesForURL(url
);
216 void LocalFileChangeTracker::DemoteChangesForURL(
217 const storage::FileSystemURL
& url
) {
218 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
220 FileChangeMap::iterator found
= changes_
.find(url
);
221 if (found
== changes_
.end())
223 DCHECK(!ContainsKey(demoted_changes_
, url
));
224 change_seqs_
.erase(found
->second
.change_seq
);
225 demoted_changes_
.insert(*found
);
226 changes_
.erase(found
);
230 void LocalFileChangeTracker::PromoteDemotedChangesForURL(
231 const storage::FileSystemURL
& url
) {
232 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
234 FileChangeMap::iterator iter
= demoted_changes_
.find(url
);
235 if (iter
== demoted_changes_
.end())
238 FileChangeList::List change_list
= iter
->second
.change_list
.list();
239 // Make sure that this URL is in no queues.
240 DCHECK(!ContainsKey(change_seqs_
, iter
->second
.change_seq
));
241 DCHECK(!ContainsKey(changes_
, url
));
243 change_seqs_
[iter
->second
.change_seq
] = url
;
244 changes_
.insert(*iter
);
245 demoted_changes_
.erase(iter
);
249 bool LocalFileChangeTracker::PromoteDemotedChanges() {
250 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
251 if (demoted_changes_
.empty())
253 while (!demoted_changes_
.empty()) {
254 storage::FileSystemURL url
= demoted_changes_
.begin()->first
;
255 PromoteDemotedChangesForURL(url
);
261 SyncStatusCode
LocalFileChangeTracker::Initialize(
262 FileSystemContext
* file_system_context
) {
263 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
264 DCHECK(!initialized_
);
265 DCHECK(file_system_context
);
267 SyncStatusCode status
= CollectLastDirtyChanges(file_system_context
);
268 if (status
== SYNC_STATUS_OK
)
273 void LocalFileChangeTracker::ResetForFileSystem(const GURL
& origin
,
274 storage::FileSystemType type
) {
275 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
276 scoped_ptr
<leveldb::WriteBatch
> batch(new leveldb::WriteBatch
);
277 for (FileChangeMap::iterator iter
= changes_
.begin();
278 iter
!= changes_
.end();) {
279 storage::FileSystemURL url
= iter
->first
;
280 int change_seq
= iter
->second
.change_seq
;
281 // Advance |iter| before calling ResetForURL to avoid the iterator
282 // invalidation in it.
284 if (url
.origin() == origin
&& url
.type() == type
)
285 ResetForURL(url
, change_seq
, batch
.get());
288 for (FileChangeMap::iterator iter
= demoted_changes_
.begin();
289 iter
!= demoted_changes_
.end();) {
290 storage::FileSystemURL url
= iter
->first
;
291 int change_seq
= iter
->second
.change_seq
;
292 // Advance |iter| before calling ResetForURL to avoid the iterator
293 // invalidation in it.
295 if (url
.origin() == origin
&& url
.type() == type
)
296 ResetForURL(url
, change_seq
, batch
.get());
299 // Fail to apply batch to database wouldn't have critical effect, they'll be
300 // just marked deleted on next relaunch.
301 tracker_db_
->WriteBatch(batch
.Pass());
305 void LocalFileChangeTracker::UpdateNumChanges() {
306 base::AutoLock
lock(num_changes_lock_
);
307 num_changes_
= static_cast<int64
>(change_seqs_
.size());
310 void LocalFileChangeTracker::GetAllChangedURLs(FileSystemURLSet
* urls
) {
311 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
312 std::deque
<FileSystemURL
> url_deque
;
313 GetNextChangedURLs(&url_deque
, 0);
315 urls
->insert(url_deque
.begin(), url_deque
.end());
318 void LocalFileChangeTracker::DropAllChanges() {
319 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
321 change_seqs_
.clear();
322 mirror_changes_
.clear();
326 SyncStatusCode
LocalFileChangeTracker::MarkDirtyOnDatabase(
327 const FileSystemURL
& url
) {
328 std::string serialized_url
;
329 if (!SerializeSyncableFileSystemURL(url
, &serialized_url
))
330 return SYNC_FILE_ERROR_INVALID_URL
;
332 return tracker_db_
->MarkDirty(serialized_url
);
335 SyncStatusCode
LocalFileChangeTracker::ClearDirtyOnDatabase(
336 const FileSystemURL
& url
) {
337 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
338 std::string serialized_url
;
339 if (!SerializeSyncableFileSystemURL(url
, &serialized_url
))
340 return SYNC_FILE_ERROR_INVALID_URL
;
342 return tracker_db_
->ClearDirty(serialized_url
);
345 SyncStatusCode
LocalFileChangeTracker::CollectLastDirtyChanges(
346 FileSystemContext
* file_system_context
) {
347 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
349 std::queue
<FileSystemURL
> dirty_files
;
350 const SyncStatusCode status
= tracker_db_
->GetDirtyEntries(&dirty_files
);
351 if (status
!= SYNC_STATUS_OK
)
354 FileSystemFileUtil
* file_util
=
355 file_system_context
->sandbox_delegate()->sync_file_util();
357 scoped_ptr
<FileSystemOperationContext
> context(
358 new FileSystemOperationContext(file_system_context
));
360 base::File::Info file_info
;
361 base::FilePath platform_path
;
363 while (!dirty_files
.empty()) {
364 const FileSystemURL url
= dirty_files
.front();
366 DCHECK_EQ(url
.type(), storage::kFileSystemTypeSyncable
);
368 switch (file_util
->GetFileInfo(context
.get(), url
,
369 &file_info
, &platform_path
)) {
370 case base::File::FILE_OK
: {
371 if (!file_info
.is_directory
) {
372 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
373 SYNC_FILE_TYPE_FILE
));
377 RecordChange(url
, FileChange(
378 FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
379 SYNC_FILE_TYPE_DIRECTORY
));
381 // Push files and directories in this directory into |dirty_files|.
382 scoped_ptr
<FileSystemFileUtil::AbstractFileEnumerator
> enumerator(
383 file_util
->CreateFileEnumerator(context
.get(), url
));
384 base::FilePath path_each
;
385 while (!(path_each
= enumerator
->Next()).empty()) {
386 dirty_files
.push(CreateSyncableFileSystemURL(
387 url
.origin(), path_each
));
391 case base::File::FILE_ERROR_NOT_FOUND
: {
392 // File represented by |url| has already been deleted. Since we cannot
393 // figure out if this file was directory or not from the URL, file
394 // type is treated as SYNC_FILE_TYPE_UNKNOWN.
396 // NOTE: Directory to have been reverted (that is, ADD -> DELETE) is
397 // also treated as FILE_CHANGE_DELETE.
398 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_DELETE
,
399 SYNC_FILE_TYPE_UNKNOWN
));
402 case base::File::FILE_ERROR_FAILED
:
404 // TODO(nhiroki): handle file access error (http://crbug.com/155251).
405 LOG(WARNING
) << "Failed to access local file.";
409 return SYNC_STATUS_OK
;
412 void LocalFileChangeTracker::RecordChange(
413 const FileSystemURL
& url
, const FileChange
& change
) {
414 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
415 int change_seq
= current_change_seq_number_
++;
416 if (ContainsKey(demoted_changes_
, url
)) {
417 RecordChangeToChangeMaps(url
, change
, change_seq
,
418 &demoted_changes_
, nullptr);
420 RecordChangeToChangeMaps(url
, change
, change_seq
, &changes_
, &change_seqs_
);
422 if (ContainsKey(mirror_changes_
, url
)) {
423 RecordChangeToChangeMaps(url
, change
, change_seq
, &mirror_changes_
,
430 void LocalFileChangeTracker::RecordChangeToChangeMaps(
431 const FileSystemURL
& url
,
432 const FileChange
& change
,
434 FileChangeMap
* changes
,
435 ChangeSeqMap
* change_seqs
) {
436 ChangeInfo
& info
= (*changes
)[url
];
437 if (info
.change_seq
>= 0 && change_seqs
)
438 change_seqs
->erase(info
.change_seq
);
439 info
.change_list
.Update(change
);
440 if (info
.change_list
.empty()) {
444 info
.change_seq
= new_change_seq
;
446 (*change_seqs
)[info
.change_seq
] = url
;
449 void LocalFileChangeTracker::ResetForURL(const storage::FileSystemURL
& url
,
451 leveldb::WriteBatch
* batch
) {
452 mirror_changes_
.erase(url
);
453 demoted_changes_
.erase(url
);
454 change_seqs_
.erase(change_seq
);
457 std::string serialized_url
;
458 if (!SerializeSyncableFileSystemURL(url
, &serialized_url
)) {
459 NOTREACHED() << "Failed to serialize: " << url
.DebugString();
462 batch
->Delete(serialized_url
);
465 // TrackerDB -------------------------------------------------------------------
467 LocalFileChangeTracker::TrackerDB::TrackerDB(const base::FilePath
& base_path
,
468 leveldb::Env
* env_override
)
469 : base_path_(base_path
),
470 env_override_(env_override
),
471 db_status_(SYNC_STATUS_OK
) {}
473 SyncStatusCode
LocalFileChangeTracker::TrackerDB::Init(
474 RecoveryOption recovery_option
) {
475 if (db_
.get() && db_status_
== SYNC_STATUS_OK
)
476 return SYNC_STATUS_OK
;
479 storage::FilePathToString(base_path_
.Append(kDatabaseName
));
480 leveldb::Options options
;
481 options
.max_open_files
= 0; // Use minimum.
482 options
.create_if_missing
= true;
484 options
.env
= env_override_
;
486 leveldb::Status status
= leveldb::DB::Open(options
, path
, &db
);
489 return SYNC_STATUS_OK
;
492 HandleError(FROM_HERE
, status
);
493 if (!status
.IsCorruption())
494 return LevelDBStatusToSyncStatusCode(status
);
496 // Try to repair the corrupted DB.
497 switch (recovery_option
) {
498 case FAIL_ON_CORRUPTION
:
499 return SYNC_DATABASE_ERROR_CORRUPTION
;
500 case REPAIR_ON_CORRUPTION
:
504 return SYNC_DATABASE_ERROR_FAILED
;
507 SyncStatusCode
LocalFileChangeTracker::TrackerDB::Repair(
508 const std::string
& db_path
) {
510 LOG(WARNING
) << "Attempting to repair TrackerDB.";
512 leveldb::Options options
;
513 options
.max_open_files
= 0; // Use minimum.
514 if (leveldb::RepairDB(db_path
, options
).ok() &&
515 Init(FAIL_ON_CORRUPTION
) == SYNC_STATUS_OK
) {
516 // TODO(nhiroki): perform some consistency checks between TrackerDB and
517 // syncable file system.
518 LOG(WARNING
) << "Repairing TrackerDB completed.";
519 return SYNC_STATUS_OK
;
522 LOG(WARNING
) << "Failed to repair TrackerDB.";
523 return SYNC_DATABASE_ERROR_CORRUPTION
;
526 // TODO(nhiroki): factor out the common methods into somewhere else.
527 void LocalFileChangeTracker::TrackerDB::HandleError(
528 const tracked_objects::Location
& from_here
,
529 const leveldb::Status
& status
) {
530 LOG(ERROR
) << "LocalFileChangeTracker::TrackerDB failed at: "
531 << from_here
.ToString() << " with error: " << status
.ToString();
534 SyncStatusCode
LocalFileChangeTracker::TrackerDB::MarkDirty(
535 const std::string
& url
) {
536 if (db_status_
!= SYNC_STATUS_OK
)
539 db_status_
= Init(REPAIR_ON_CORRUPTION
);
540 if (db_status_
!= SYNC_STATUS_OK
) {
545 leveldb::Status status
= db_
->Put(leveldb::WriteOptions(), url
, kMark
);
547 HandleError(FROM_HERE
, status
);
548 db_status_
= LevelDBStatusToSyncStatusCode(status
);
552 return SYNC_STATUS_OK
;
555 SyncStatusCode
LocalFileChangeTracker::TrackerDB::ClearDirty(
556 const std::string
& url
) {
557 if (db_status_
!= SYNC_STATUS_OK
)
560 // Should not reach here before initializing the database. The database should
561 // be cleared after read, and should be initialized during read if
565 leveldb::Status status
= db_
->Delete(leveldb::WriteOptions(), url
);
566 if (!status
.ok() && !status
.IsNotFound()) {
567 HandleError(FROM_HERE
, status
);
568 db_status_
= LevelDBStatusToSyncStatusCode(status
);
572 return SYNC_STATUS_OK
;
575 SyncStatusCode
LocalFileChangeTracker::TrackerDB::GetDirtyEntries(
576 std::queue
<FileSystemURL
>* dirty_files
) {
577 if (db_status_
!= SYNC_STATUS_OK
)
580 db_status_
= Init(REPAIR_ON_CORRUPTION
);
581 if (db_status_
!= SYNC_STATUS_OK
) {
586 scoped_ptr
<leveldb::Iterator
> iter(db_
->NewIterator(leveldb::ReadOptions()));
589 while (iter
->Valid()) {
590 if (!DeserializeSyncableFileSystemURL(iter
->key().ToString(), &url
)) {
591 LOG(WARNING
) << "Failed to deserialize an URL. "
592 << "TrackerDB might be corrupted.";
593 db_status_
= SYNC_DATABASE_ERROR_CORRUPTION
;
597 dirty_files
->push(url
);
600 return SYNC_STATUS_OK
;
603 SyncStatusCode
LocalFileChangeTracker::TrackerDB::WriteBatch(
604 scoped_ptr
<leveldb::WriteBatch
> batch
) {
605 if (db_status_
!= SYNC_STATUS_OK
)
608 leveldb::Status status
= db_
->Write(leveldb::WriteOptions(), batch
.get());
609 if (!status
.ok() && !status
.IsNotFound()) {
610 HandleError(FROM_HERE
, status
);
611 db_status_
= LevelDBStatusToSyncStatusCode(status
);
615 return SYNC_STATUS_OK
;
618 } // namespace sync_file_system