1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/sync_file_system/local/local_file_change_tracker.h"
9 #include "base/location.h"
10 #include "base/logging.h"
11 #include "base/sequenced_task_runner.h"
12 #include "base/stl_util.h"
13 #include "chrome/browser/sync_file_system/local/local_file_sync_status.h"
14 #include "chrome/browser/sync_file_system/syncable_file_system_util.h"
15 #include "storage/browser/fileapi/file_system_context.h"
16 #include "storage/browser/fileapi/file_system_file_util.h"
17 #include "storage/browser/fileapi/file_system_operation_context.h"
18 #include "storage/common/fileapi/file_system_util.h"
19 #include "third_party/leveldatabase/env_chromium.h"
20 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
21 #include "third_party/leveldatabase/src/include/leveldb/db.h"
22 #include "third_party/leveldatabase/src/include/leveldb/env.h"
23 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
25 using storage::FileSystemContext
;
26 using storage::FileSystemFileUtil
;
27 using storage::FileSystemOperationContext
;
28 using storage::FileSystemURL
;
29 using storage::FileSystemURLSet
;
31 namespace sync_file_system
{
34 const base::FilePath::CharType kDatabaseName
[] =
35 FILE_PATH_LITERAL("LocalFileChangeTracker");
36 const char kMark
[] = "d";
39 // A database class that stores local file changes in a local database. This
40 // object must be destructed on file_task_runner.
41 class LocalFileChangeTracker::TrackerDB
{
43 TrackerDB(const base::FilePath
& base_path
,
44 leveldb::Env
* env_override
);
46 SyncStatusCode
MarkDirty(const std::string
& url
);
47 SyncStatusCode
ClearDirty(const std::string
& url
);
48 SyncStatusCode
GetDirtyEntries(
49 std::queue
<FileSystemURL
>* dirty_files
);
50 SyncStatusCode
WriteBatch(scoped_ptr
<leveldb::WriteBatch
> batch
);
58 SyncStatusCode
Init(RecoveryOption recovery_option
);
59 SyncStatusCode
Repair(const std::string
& db_path
);
60 void HandleError(const tracked_objects::Location
& from_here
,
61 const leveldb::Status
& status
);
63 const base::FilePath base_path_
;
64 leveldb::Env
* env_override_
;
65 scoped_ptr
<leveldb::DB
> db_
;
66 SyncStatusCode db_status_
;
68 DISALLOW_COPY_AND_ASSIGN(TrackerDB
);
71 LocalFileChangeTracker::ChangeInfo::ChangeInfo() : change_seq(-1) {}
72 LocalFileChangeTracker::ChangeInfo::~ChangeInfo() {}
74 // LocalFileChangeTracker ------------------------------------------------------
76 LocalFileChangeTracker::LocalFileChangeTracker(
77 const base::FilePath
& base_path
,
78 leveldb::Env
* env_override
,
79 base::SequencedTaskRunner
* file_task_runner
)
80 : initialized_(false),
81 file_task_runner_(file_task_runner
),
82 tracker_db_(new TrackerDB(base_path
, env_override
)),
83 current_change_seq_number_(0),
87 LocalFileChangeTracker::~LocalFileChangeTracker() {
88 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
92 void LocalFileChangeTracker::OnStartUpdate(const FileSystemURL
& url
) {
93 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
94 if (ContainsKey(changes_
, url
) || ContainsKey(demoted_changes_
, url
))
96 // TODO(nhiroki): propagate the error code (see http://crbug.com/152127).
97 MarkDirtyOnDatabase(url
);
100 void LocalFileChangeTracker::OnEndUpdate(const FileSystemURL
& url
) {}
102 void LocalFileChangeTracker::OnCreateFile(const FileSystemURL
& url
) {
103 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
104 SYNC_FILE_TYPE_FILE
));
107 void LocalFileChangeTracker::OnCreateFileFrom(const FileSystemURL
& url
,
108 const FileSystemURL
& src
) {
109 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
110 SYNC_FILE_TYPE_FILE
));
113 void LocalFileChangeTracker::OnRemoveFile(const FileSystemURL
& url
) {
114 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_DELETE
,
115 SYNC_FILE_TYPE_FILE
));
118 void LocalFileChangeTracker::OnModifyFile(const FileSystemURL
& url
) {
119 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
120 SYNC_FILE_TYPE_FILE
));
123 void LocalFileChangeTracker::OnCreateDirectory(const FileSystemURL
& url
) {
124 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
125 SYNC_FILE_TYPE_DIRECTORY
));
128 void LocalFileChangeTracker::OnRemoveDirectory(const FileSystemURL
& url
) {
129 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_DELETE
,
130 SYNC_FILE_TYPE_DIRECTORY
));
133 void LocalFileChangeTracker::GetNextChangedURLs(
134 std::deque
<FileSystemURL
>* urls
, int max_urls
) {
135 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
138 // Mildly prioritizes the URLs that older changes and have not been updated
140 for (ChangeSeqMap::iterator iter
= change_seqs_
.begin();
141 iter
!= change_seqs_
.end() &&
142 (max_urls
== 0 || urls
->size() < static_cast<size_t>(max_urls
));
144 urls
->push_back(iter
->second
);
148 void LocalFileChangeTracker::GetChangesForURL(
149 const FileSystemURL
& url
, FileChangeList
* changes
) {
150 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
153 FileChangeMap::iterator found
= changes_
.find(url
);
154 if (found
== changes_
.end()) {
155 found
= demoted_changes_
.find(url
);
156 if (found
== demoted_changes_
.end())
159 *changes
= found
->second
.change_list
;
162 void LocalFileChangeTracker::ClearChangesForURL(const FileSystemURL
& url
) {
163 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
164 ClearDirtyOnDatabase(url
);
165 mirror_changes_
.erase(url
);
166 demoted_changes_
.erase(url
);
167 FileChangeMap::iterator found
= changes_
.find(url
);
168 if (found
== changes_
.end())
170 change_seqs_
.erase(found
->second
.change_seq
);
171 changes_
.erase(found
);
175 void LocalFileChangeTracker::CreateFreshMirrorForURL(
176 const storage::FileSystemURL
& url
) {
177 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
178 DCHECK(!ContainsKey(mirror_changes_
, url
));
179 mirror_changes_
[url
] = ChangeInfo();
182 void LocalFileChangeTracker::RemoveMirrorAndCommitChangesForURL(
183 const storage::FileSystemURL
& url
) {
184 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
185 FileChangeMap::iterator found
= mirror_changes_
.find(url
);
186 if (found
== mirror_changes_
.end())
188 mirror_changes_
.erase(found
);
190 if (ContainsKey(changes_
, url
) || ContainsKey(demoted_changes_
, url
))
191 MarkDirtyOnDatabase(url
);
193 ClearDirtyOnDatabase(url
);
197 void LocalFileChangeTracker::ResetToMirrorAndCommitChangesForURL(
198 const storage::FileSystemURL
& url
) {
199 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
200 FileChangeMap::iterator found
= mirror_changes_
.find(url
);
201 if (found
== mirror_changes_
.end() || found
->second
.change_list
.empty()) {
202 ClearChangesForURL(url
);
205 const ChangeInfo
& info
= found
->second
;
206 if (ContainsKey(demoted_changes_
, url
)) {
207 DCHECK(!ContainsKey(changes_
, url
));
208 demoted_changes_
[url
] = info
;
210 DCHECK(!ContainsKey(demoted_changes_
, url
));
211 change_seqs_
[info
.change_seq
] = url
;
212 changes_
[url
] = info
;
214 RemoveMirrorAndCommitChangesForURL(url
);
217 void LocalFileChangeTracker::DemoteChangesForURL(
218 const storage::FileSystemURL
& url
) {
219 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
221 FileChangeMap::iterator found
= changes_
.find(url
);
222 if (found
== changes_
.end())
224 DCHECK(!ContainsKey(demoted_changes_
, url
));
225 change_seqs_
.erase(found
->second
.change_seq
);
226 demoted_changes_
.insert(*found
);
227 changes_
.erase(found
);
231 void LocalFileChangeTracker::PromoteDemotedChangesForURL(
232 const storage::FileSystemURL
& url
) {
233 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
235 FileChangeMap::iterator iter
= demoted_changes_
.find(url
);
236 if (iter
== demoted_changes_
.end())
239 FileChangeList::List change_list
= iter
->second
.change_list
.list();
240 // Make sure that this URL is in no queues.
241 DCHECK(!ContainsKey(change_seqs_
, iter
->second
.change_seq
));
242 DCHECK(!ContainsKey(changes_
, url
));
244 change_seqs_
[iter
->second
.change_seq
] = url
;
245 changes_
.insert(*iter
);
246 demoted_changes_
.erase(iter
);
250 bool LocalFileChangeTracker::PromoteDemotedChanges() {
251 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
252 if (demoted_changes_
.empty())
254 while (!demoted_changes_
.empty()) {
255 storage::FileSystemURL url
= demoted_changes_
.begin()->first
;
256 PromoteDemotedChangesForURL(url
);
262 SyncStatusCode
LocalFileChangeTracker::Initialize(
263 FileSystemContext
* file_system_context
) {
264 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
265 DCHECK(!initialized_
);
266 DCHECK(file_system_context
);
268 SyncStatusCode status
= CollectLastDirtyChanges(file_system_context
);
269 if (status
== SYNC_STATUS_OK
)
274 void LocalFileChangeTracker::ResetForFileSystem(const GURL
& origin
,
275 storage::FileSystemType type
) {
276 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
277 scoped_ptr
<leveldb::WriteBatch
> batch(new leveldb::WriteBatch
);
278 for (FileChangeMap::iterator iter
= changes_
.begin();
279 iter
!= changes_
.end();) {
280 storage::FileSystemURL url
= iter
->first
;
281 int change_seq
= iter
->second
.change_seq
;
282 // Advance |iter| before calling ResetForURL to avoid the iterator
283 // invalidation in it.
285 if (url
.origin() == origin
&& url
.type() == type
)
286 ResetForURL(url
, change_seq
, batch
.get());
289 for (FileChangeMap::iterator iter
= demoted_changes_
.begin();
290 iter
!= demoted_changes_
.end();) {
291 storage::FileSystemURL url
= iter
->first
;
292 int change_seq
= iter
->second
.change_seq
;
293 // Advance |iter| before calling ResetForURL to avoid the iterator
294 // invalidation in it.
296 if (url
.origin() == origin
&& url
.type() == type
)
297 ResetForURL(url
, change_seq
, batch
.get());
300 // Fail to apply batch to database wouldn't have critical effect, they'll be
301 // just marked deleted on next relaunch.
302 tracker_db_
->WriteBatch(batch
.Pass());
306 void LocalFileChangeTracker::UpdateNumChanges() {
307 base::AutoLock
lock(num_changes_lock_
);
308 num_changes_
= static_cast<int64
>(change_seqs_
.size());
311 void LocalFileChangeTracker::GetAllChangedURLs(FileSystemURLSet
* urls
) {
312 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
313 std::deque
<FileSystemURL
> url_deque
;
314 GetNextChangedURLs(&url_deque
, 0);
316 urls
->insert(url_deque
.begin(), url_deque
.end());
319 void LocalFileChangeTracker::DropAllChanges() {
320 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
322 change_seqs_
.clear();
323 mirror_changes_
.clear();
327 SyncStatusCode
LocalFileChangeTracker::MarkDirtyOnDatabase(
328 const FileSystemURL
& url
) {
329 std::string serialized_url
;
330 if (!SerializeSyncableFileSystemURL(url
, &serialized_url
))
331 return SYNC_FILE_ERROR_INVALID_URL
;
333 return tracker_db_
->MarkDirty(serialized_url
);
336 SyncStatusCode
LocalFileChangeTracker::ClearDirtyOnDatabase(
337 const FileSystemURL
& url
) {
338 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
339 std::string serialized_url
;
340 if (!SerializeSyncableFileSystemURL(url
, &serialized_url
))
341 return SYNC_FILE_ERROR_INVALID_URL
;
343 return tracker_db_
->ClearDirty(serialized_url
);
346 SyncStatusCode
LocalFileChangeTracker::CollectLastDirtyChanges(
347 FileSystemContext
* file_system_context
) {
348 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
350 std::queue
<FileSystemURL
> dirty_files
;
351 const SyncStatusCode status
= tracker_db_
->GetDirtyEntries(&dirty_files
);
352 if (status
!= SYNC_STATUS_OK
)
355 FileSystemFileUtil
* file_util
=
356 file_system_context
->sandbox_delegate()->sync_file_util();
358 scoped_ptr
<FileSystemOperationContext
> context(
359 new FileSystemOperationContext(file_system_context
));
361 base::File::Info file_info
;
362 base::FilePath platform_path
;
364 while (!dirty_files
.empty()) {
365 const FileSystemURL url
= dirty_files
.front();
367 DCHECK_EQ(url
.type(), storage::kFileSystemTypeSyncable
);
369 switch (file_util
->GetFileInfo(context
.get(), url
,
370 &file_info
, &platform_path
)) {
371 case base::File::FILE_OK
: {
372 if (!file_info
.is_directory
) {
373 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
374 SYNC_FILE_TYPE_FILE
));
378 RecordChange(url
, FileChange(
379 FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
380 SYNC_FILE_TYPE_DIRECTORY
));
382 // Push files and directories in this directory into |dirty_files|.
383 scoped_ptr
<FileSystemFileUtil::AbstractFileEnumerator
> enumerator(
384 file_util
->CreateFileEnumerator(context
.get(), url
));
385 base::FilePath path_each
;
386 while (!(path_each
= enumerator
->Next()).empty()) {
387 dirty_files
.push(CreateSyncableFileSystemURL(
388 url
.origin(), path_each
));
392 case base::File::FILE_ERROR_NOT_FOUND
: {
393 // File represented by |url| has already been deleted. Since we cannot
394 // figure out if this file was directory or not from the URL, file
395 // type is treated as SYNC_FILE_TYPE_UNKNOWN.
397 // NOTE: Directory to have been reverted (that is, ADD -> DELETE) is
398 // also treated as FILE_CHANGE_DELETE.
399 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_DELETE
,
400 SYNC_FILE_TYPE_UNKNOWN
));
403 case base::File::FILE_ERROR_FAILED
:
405 // TODO(nhiroki): handle file access error (http://crbug.com/155251).
406 LOG(WARNING
) << "Failed to access local file.";
410 return SYNC_STATUS_OK
;
413 void LocalFileChangeTracker::RecordChange(
414 const FileSystemURL
& url
, const FileChange
& change
) {
415 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
416 int change_seq
= current_change_seq_number_
++;
417 if (ContainsKey(demoted_changes_
, url
)) {
418 RecordChangeToChangeMaps(url
, change
, change_seq
,
419 &demoted_changes_
, nullptr);
421 RecordChangeToChangeMaps(url
, change
, change_seq
, &changes_
, &change_seqs_
);
423 if (ContainsKey(mirror_changes_
, url
)) {
424 RecordChangeToChangeMaps(url
, change
, change_seq
, &mirror_changes_
,
431 void LocalFileChangeTracker::RecordChangeToChangeMaps(
432 const FileSystemURL
& url
,
433 const FileChange
& change
,
435 FileChangeMap
* changes
,
436 ChangeSeqMap
* change_seqs
) {
437 ChangeInfo
& info
= (*changes
)[url
];
438 if (info
.change_seq
>= 0 && change_seqs
)
439 change_seqs
->erase(info
.change_seq
);
440 info
.change_list
.Update(change
);
441 if (info
.change_list
.empty()) {
445 info
.change_seq
= new_change_seq
;
447 (*change_seqs
)[info
.change_seq
] = url
;
450 void LocalFileChangeTracker::ResetForURL(const storage::FileSystemURL
& url
,
452 leveldb::WriteBatch
* batch
) {
453 mirror_changes_
.erase(url
);
454 demoted_changes_
.erase(url
);
455 change_seqs_
.erase(change_seq
);
458 std::string serialized_url
;
459 if (!SerializeSyncableFileSystemURL(url
, &serialized_url
)) {
460 NOTREACHED() << "Failed to serialize: " << url
.DebugString();
463 batch
->Delete(serialized_url
);
466 // TrackerDB -------------------------------------------------------------------
468 LocalFileChangeTracker::TrackerDB::TrackerDB(const base::FilePath
& base_path
,
469 leveldb::Env
* env_override
)
470 : base_path_(base_path
),
471 env_override_(env_override
),
472 db_status_(SYNC_STATUS_OK
) {}
474 SyncStatusCode
LocalFileChangeTracker::TrackerDB::Init(
475 RecoveryOption recovery_option
) {
476 if (db_
.get() && db_status_
== SYNC_STATUS_OK
)
477 return SYNC_STATUS_OK
;
480 storage::FilePathToString(base_path_
.Append(kDatabaseName
));
481 leveldb::Options options
;
482 options
.max_open_files
= 0; // Use minimum.
483 options
.create_if_missing
= true;
484 options
.reuse_logs
= leveldb_env::kDefaultLogReuseOptionValue
;
486 options
.env
= env_override_
;
488 leveldb::Status status
= leveldb::DB::Open(options
, path
, &db
);
491 return SYNC_STATUS_OK
;
494 HandleError(FROM_HERE
, status
);
495 if (!status
.IsCorruption())
496 return LevelDBStatusToSyncStatusCode(status
);
498 // Try to repair the corrupted DB.
499 switch (recovery_option
) {
500 case FAIL_ON_CORRUPTION
:
501 return SYNC_DATABASE_ERROR_CORRUPTION
;
502 case REPAIR_ON_CORRUPTION
:
506 return SYNC_DATABASE_ERROR_FAILED
;
509 SyncStatusCode
LocalFileChangeTracker::TrackerDB::Repair(
510 const std::string
& db_path
) {
512 LOG(WARNING
) << "Attempting to repair TrackerDB.";
514 leveldb::Options options
;
515 options
.max_open_files
= 0; // Use minimum.
516 if (leveldb::RepairDB(db_path
, options
).ok() &&
517 Init(FAIL_ON_CORRUPTION
) == SYNC_STATUS_OK
) {
518 // TODO(nhiroki): perform some consistency checks between TrackerDB and
519 // syncable file system.
520 LOG(WARNING
) << "Repairing TrackerDB completed.";
521 return SYNC_STATUS_OK
;
524 LOG(WARNING
) << "Failed to repair TrackerDB.";
525 return SYNC_DATABASE_ERROR_CORRUPTION
;
528 // TODO(nhiroki): factor out the common methods into somewhere else.
529 void LocalFileChangeTracker::TrackerDB::HandleError(
530 const tracked_objects::Location
& from_here
,
531 const leveldb::Status
& status
) {
532 LOG(ERROR
) << "LocalFileChangeTracker::TrackerDB failed at: "
533 << from_here
.ToString() << " with error: " << status
.ToString();
536 SyncStatusCode
LocalFileChangeTracker::TrackerDB::MarkDirty(
537 const std::string
& url
) {
538 if (db_status_
!= SYNC_STATUS_OK
)
541 db_status_
= Init(REPAIR_ON_CORRUPTION
);
542 if (db_status_
!= SYNC_STATUS_OK
) {
547 leveldb::Status status
= db_
->Put(leveldb::WriteOptions(), url
, kMark
);
549 HandleError(FROM_HERE
, status
);
550 db_status_
= LevelDBStatusToSyncStatusCode(status
);
554 return SYNC_STATUS_OK
;
557 SyncStatusCode
LocalFileChangeTracker::TrackerDB::ClearDirty(
558 const std::string
& url
) {
559 if (db_status_
!= SYNC_STATUS_OK
)
562 // Should not reach here before initializing the database. The database should
563 // be cleared after read, and should be initialized during read if
567 leveldb::Status status
= db_
->Delete(leveldb::WriteOptions(), url
);
568 if (!status
.ok() && !status
.IsNotFound()) {
569 HandleError(FROM_HERE
, status
);
570 db_status_
= LevelDBStatusToSyncStatusCode(status
);
574 return SYNC_STATUS_OK
;
577 SyncStatusCode
LocalFileChangeTracker::TrackerDB::GetDirtyEntries(
578 std::queue
<FileSystemURL
>* dirty_files
) {
579 if (db_status_
!= SYNC_STATUS_OK
)
582 db_status_
= Init(REPAIR_ON_CORRUPTION
);
583 if (db_status_
!= SYNC_STATUS_OK
) {
588 scoped_ptr
<leveldb::Iterator
> iter(db_
->NewIterator(leveldb::ReadOptions()));
591 while (iter
->Valid()) {
592 if (!DeserializeSyncableFileSystemURL(iter
->key().ToString(), &url
)) {
593 LOG(WARNING
) << "Failed to deserialize an URL. "
594 << "TrackerDB might be corrupted.";
595 db_status_
= SYNC_DATABASE_ERROR_CORRUPTION
;
599 dirty_files
->push(url
);
602 return SYNC_STATUS_OK
;
605 SyncStatusCode
LocalFileChangeTracker::TrackerDB::WriteBatch(
606 scoped_ptr
<leveldb::WriteBatch
> batch
) {
607 if (db_status_
!= SYNC_STATUS_OK
)
610 leveldb::Status status
= db_
->Write(leveldb::WriteOptions(), batch
.get());
611 if (!status
.ok() && !status
.IsNotFound()) {
612 HandleError(FROM_HERE
, status
);
613 db_status_
= LevelDBStatusToSyncStatusCode(status
);
617 return SYNC_STATUS_OK
;
620 } // namespace sync_file_system