1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/sync_file_system/local/local_file_change_tracker.h"
9 #include "base/location.h"
10 #include "base/logging.h"
11 #include "base/sequenced_task_runner.h"
12 #include "base/stl_util.h"
13 #include "chrome/browser/sync_file_system/local/local_file_sync_status.h"
14 #include "chrome/browser/sync_file_system/syncable_file_system_util.h"
15 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
16 #include "third_party/leveldatabase/src/include/leveldb/db.h"
17 #include "third_party/leveldatabase/src/include/leveldb/env.h"
18 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
19 #include "webkit/browser/fileapi/file_system_context.h"
20 #include "webkit/browser/fileapi/file_system_file_util.h"
21 #include "webkit/browser/fileapi/file_system_operation_context.h"
22 #include "webkit/common/fileapi/file_system_util.h"
24 using fileapi::FileSystemContext
;
25 using fileapi::FileSystemFileUtil
;
26 using fileapi::FileSystemOperationContext
;
27 using fileapi::FileSystemURL
;
28 using fileapi::FileSystemURLSet
;
30 namespace sync_file_system
{
33 const base::FilePath::CharType kDatabaseName
[] =
34 FILE_PATH_LITERAL("LocalFileChangeTracker");
35 const char kMark
[] = "d";
38 // A database class that stores local file changes in a local database. This
39 // object must be destructed on file_task_runner.
40 class LocalFileChangeTracker::TrackerDB
{
42 TrackerDB(const base::FilePath
& base_path
,
43 leveldb::Env
* env_override
);
45 SyncStatusCode
MarkDirty(const std::string
& url
);
46 SyncStatusCode
ClearDirty(const std::string
& url
);
47 SyncStatusCode
GetDirtyEntries(
48 std::queue
<FileSystemURL
>* dirty_files
);
49 SyncStatusCode
WriteBatch(scoped_ptr
<leveldb::WriteBatch
> batch
);
57 SyncStatusCode
Init(RecoveryOption recovery_option
);
58 SyncStatusCode
Repair(const std::string
& db_path
);
59 void HandleError(const tracked_objects::Location
& from_here
,
60 const leveldb::Status
& status
);
62 const base::FilePath base_path_
;
63 leveldb::Env
* env_override_
;
64 scoped_ptr
<leveldb::DB
> db_
;
65 SyncStatusCode db_status_
;
67 DISALLOW_COPY_AND_ASSIGN(TrackerDB
);
70 LocalFileChangeTracker::ChangeInfo::ChangeInfo() : change_seq(-1) {}
71 LocalFileChangeTracker::ChangeInfo::~ChangeInfo() {}
73 // LocalFileChangeTracker ------------------------------------------------------
75 LocalFileChangeTracker::LocalFileChangeTracker(
76 const base::FilePath
& base_path
,
77 leveldb::Env
* env_override
,
78 base::SequencedTaskRunner
* file_task_runner
)
79 : initialized_(false),
80 file_task_runner_(file_task_runner
),
81 tracker_db_(new TrackerDB(base_path
, env_override
)),
82 current_change_seq_(0),
86 LocalFileChangeTracker::~LocalFileChangeTracker() {
87 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
91 void LocalFileChangeTracker::OnStartUpdate(const FileSystemURL
& url
) {
92 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
93 if (ContainsKey(changes_
, url
))
95 // TODO(nhiroki): propagate the error code (see http://crbug.com/152127).
96 MarkDirtyOnDatabase(url
);
99 void LocalFileChangeTracker::OnEndUpdate(const FileSystemURL
& url
) {}
101 void LocalFileChangeTracker::OnCreateFile(const FileSystemURL
& url
) {
102 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
103 SYNC_FILE_TYPE_FILE
));
106 void LocalFileChangeTracker::OnCreateFileFrom(const FileSystemURL
& url
,
107 const FileSystemURL
& src
) {
108 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
109 SYNC_FILE_TYPE_FILE
));
112 void LocalFileChangeTracker::OnRemoveFile(const FileSystemURL
& url
) {
113 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_DELETE
,
114 SYNC_FILE_TYPE_FILE
));
117 void LocalFileChangeTracker::OnModifyFile(const FileSystemURL
& url
) {
118 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
119 SYNC_FILE_TYPE_FILE
));
122 void LocalFileChangeTracker::OnCreateDirectory(const FileSystemURL
& url
) {
123 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
124 SYNC_FILE_TYPE_DIRECTORY
));
127 void LocalFileChangeTracker::OnRemoveDirectory(const FileSystemURL
& url
) {
128 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_DELETE
,
129 SYNC_FILE_TYPE_DIRECTORY
));
132 void LocalFileChangeTracker::GetNextChangedURLs(
133 std::deque
<FileSystemURL
>* urls
, int max_urls
) {
135 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
137 // Mildly prioritizes the URLs that older changes and have not been updated
139 for (ChangeSeqMap::iterator iter
= change_seqs_
.begin();
140 iter
!= change_seqs_
.end() &&
141 (max_urls
== 0 || urls
->size() < static_cast<size_t>(max_urls
));
143 urls
->push_back(iter
->second
);
147 void LocalFileChangeTracker::GetChangesForURL(
148 const FileSystemURL
& url
, FileChangeList
* changes
) {
149 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
152 FileChangeMap::iterator found
= changes_
.find(url
);
153 if (found
== changes_
.end()) {
154 found
= demoted_changes_
.find(url
);
155 if (found
== demoted_changes_
.end())
158 *changes
= found
->second
.change_list
;
161 void LocalFileChangeTracker::ClearChangesForURL(const FileSystemURL
& url
) {
162 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
163 ClearDirtyOnDatabase(url
);
164 mirror_changes_
.erase(url
);
165 demoted_changes_
.erase(url
);
166 FileChangeMap::iterator found
= changes_
.find(url
);
167 if (found
== changes_
.end())
169 change_seqs_
.erase(found
->second
.change_seq
);
170 changes_
.erase(found
);
174 void LocalFileChangeTracker::CreateFreshMirrorForURL(
175 const fileapi::FileSystemURL
& url
) {
176 DCHECK(!ContainsKey(mirror_changes_
, url
));
177 mirror_changes_
[url
] = ChangeInfo();
180 void LocalFileChangeTracker::RemoveMirrorAndCommitChangesForURL(
181 const fileapi::FileSystemURL
& url
) {
182 FileChangeMap::iterator found
= mirror_changes_
.find(url
);
183 if (found
== mirror_changes_
.end())
185 mirror_changes_
.erase(found
);
187 if (ContainsKey(changes_
, url
))
188 MarkDirtyOnDatabase(url
);
190 ClearDirtyOnDatabase(url
);
194 void LocalFileChangeTracker::ResetToMirrorAndCommitChangesForURL(
195 const fileapi::FileSystemURL
& url
) {
196 FileChangeMap::iterator found
= mirror_changes_
.find(url
);
197 if (found
== mirror_changes_
.end() || found
->second
.change_list
.empty()) {
198 ClearChangesForURL(url
);
201 const ChangeInfo
& info
= found
->second
;
202 change_seqs_
[info
.change_seq
] = url
;
203 changes_
[url
] = info
;
204 RemoveMirrorAndCommitChangesForURL(url
);
207 void LocalFileChangeTracker::DemoteChangesForURL(
208 const fileapi::FileSystemURL
& url
) {
209 FileChangeMap::iterator found
= changes_
.find(url
);
210 if (found
== changes_
.end())
212 FileChangeList changes
= found
->second
.change_list
;
214 mirror_changes_
.erase(url
);
215 change_seqs_
.erase(found
->second
.change_seq
);
216 changes_
.erase(found
);
218 FileChangeList::List change_list
= changes
.list();
219 while (!change_list
.empty()) {
220 RecordChangeToChangeMaps(url
, change_list
.front(), 0,
221 &demoted_changes_
, NULL
);
222 change_list
.pop_front();
226 bool LocalFileChangeTracker::PromoteDemotedChanges() {
227 if (demoted_changes_
.empty())
229 for (FileChangeMap::iterator iter
= demoted_changes_
.begin();
230 iter
!= demoted_changes_
.end();) {
231 fileapi::FileSystemURL url
= iter
->first
;
232 FileChangeList::List change_list
= iter
->second
.change_list
.list();
233 demoted_changes_
.erase(iter
++);
235 // Make sure that this URL is in no queues.
236 DCHECK(!ContainsKey(changes_
, url
));
237 DCHECK(!ContainsKey(demoted_changes_
, url
));
238 DCHECK(!ContainsKey(mirror_changes_
, url
));
240 while (!change_list
.empty()) {
241 RecordChange(url
, change_list
.front());
242 change_list
.pop_front();
245 demoted_changes_
.clear();
249 SyncStatusCode
LocalFileChangeTracker::Initialize(
250 FileSystemContext
* file_system_context
) {
251 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
252 DCHECK(!initialized_
);
253 DCHECK(file_system_context
);
255 SyncStatusCode status
= CollectLastDirtyChanges(file_system_context
);
256 if (status
== SYNC_STATUS_OK
)
261 void LocalFileChangeTracker::ResetForFileSystem(
263 fileapi::FileSystemType type
) {
264 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
265 scoped_ptr
<leveldb::WriteBatch
> batch(new leveldb::WriteBatch
);
266 for (FileChangeMap::iterator iter
= changes_
.begin();
267 iter
!= changes_
.end();) {
268 fileapi::FileSystemURL url
= iter
->first
;
269 if (url
.origin() != origin
|| url
.type() != type
) {
273 mirror_changes_
.erase(url
);
274 demoted_changes_
.erase(url
);
275 change_seqs_
.erase(iter
->second
.change_seq
);
276 changes_
.erase(iter
++);
278 std::string serialized_url
;
279 const bool should_success
=
280 SerializeSyncableFileSystemURL(url
, &serialized_url
);
281 if (!should_success
) {
282 NOTREACHED() << "Failed to serialize: " << url
.DebugString();
285 batch
->Delete(serialized_url
);
287 // Fail to apply batch to database wouldn't have critical effect, they'll be
288 // just marked deleted on next relaunch.
289 tracker_db_
->WriteBatch(batch
.Pass());
293 void LocalFileChangeTracker::UpdateNumChanges() {
294 base::AutoLock
lock(num_changes_lock_
);
295 num_changes_
= static_cast<int64
>(change_seqs_
.size());
298 void LocalFileChangeTracker::GetAllChangedURLs(FileSystemURLSet
* urls
) {
299 std::deque
<FileSystemURL
> url_deque
;
300 GetNextChangedURLs(&url_deque
, 0);
302 urls
->insert(url_deque
.begin(), url_deque
.end());
305 void LocalFileChangeTracker::DropAllChanges() {
307 change_seqs_
.clear();
308 mirror_changes_
.clear();
311 SyncStatusCode
LocalFileChangeTracker::MarkDirtyOnDatabase(
312 const FileSystemURL
& url
) {
313 std::string serialized_url
;
314 if (!SerializeSyncableFileSystemURL(url
, &serialized_url
))
315 return SYNC_FILE_ERROR_INVALID_URL
;
317 return tracker_db_
->MarkDirty(serialized_url
);
320 SyncStatusCode
LocalFileChangeTracker::ClearDirtyOnDatabase(
321 const FileSystemURL
& url
) {
322 std::string serialized_url
;
323 if (!SerializeSyncableFileSystemURL(url
, &serialized_url
))
324 return SYNC_FILE_ERROR_INVALID_URL
;
326 return tracker_db_
->ClearDirty(serialized_url
);
329 SyncStatusCode
LocalFileChangeTracker::CollectLastDirtyChanges(
330 FileSystemContext
* file_system_context
) {
331 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
333 std::queue
<FileSystemURL
> dirty_files
;
334 const SyncStatusCode status
= tracker_db_
->GetDirtyEntries(&dirty_files
);
335 if (status
!= SYNC_STATUS_OK
)
338 FileSystemFileUtil
* file_util
=
339 file_system_context
->sandbox_delegate()->sync_file_util();
341 scoped_ptr
<FileSystemOperationContext
> context(
342 new FileSystemOperationContext(file_system_context
));
344 base::File::Info file_info
;
345 base::FilePath platform_path
;
347 while (!dirty_files
.empty()) {
348 const FileSystemURL url
= dirty_files
.front();
350 DCHECK_EQ(url
.type(), fileapi::kFileSystemTypeSyncable
);
352 switch (file_util
->GetFileInfo(context
.get(), url
,
353 &file_info
, &platform_path
)) {
354 case base::PLATFORM_FILE_OK
: {
355 if (!file_info
.is_directory
) {
356 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
357 SYNC_FILE_TYPE_FILE
));
361 RecordChange(url
, FileChange(
362 FileChange::FILE_CHANGE_ADD_OR_UPDATE
,
363 SYNC_FILE_TYPE_DIRECTORY
));
365 // Push files and directories in this directory into |dirty_files|.
366 scoped_ptr
<FileSystemFileUtil::AbstractFileEnumerator
> enumerator(
367 file_util
->CreateFileEnumerator(context
.get(), url
));
368 base::FilePath path_each
;
369 while (!(path_each
= enumerator
->Next()).empty()) {
370 dirty_files
.push(CreateSyncableFileSystemURL(
371 url
.origin(), path_each
));
375 case base::PLATFORM_FILE_ERROR_NOT_FOUND
: {
376 // File represented by |url| has already been deleted. Since we cannot
377 // figure out if this file was directory or not from the URL, file
378 // type is treated as SYNC_FILE_TYPE_UNKNOWN.
380 // NOTE: Directory to have been reverted (that is, ADD -> DELETE) is
381 // also treated as FILE_CHANGE_DELETE.
382 RecordChange(url
, FileChange(FileChange::FILE_CHANGE_DELETE
,
383 SYNC_FILE_TYPE_UNKNOWN
));
386 case base::PLATFORM_FILE_ERROR_FAILED
:
388 // TODO(nhiroki): handle file access error (http://crbug.com/155251).
389 LOG(WARNING
) << "Failed to access local file.";
393 return SYNC_STATUS_OK
;
396 void LocalFileChangeTracker::RecordChange(
397 const FileSystemURL
& url
, const FileChange
& change
) {
398 DCHECK(file_task_runner_
->RunsTasksOnCurrentThread());
399 if (ContainsKey(demoted_changes_
, url
)) {
400 RecordChangeToChangeMaps(url
, change
, 0, &demoted_changes_
, NULL
);
403 int change_seq
= current_change_seq_
++;
404 RecordChangeToChangeMaps(url
, change
, change_seq
, &changes_
, &change_seqs_
);
405 if (ContainsKey(mirror_changes_
, url
))
406 RecordChangeToChangeMaps(url
, change
, change_seq
, &mirror_changes_
, NULL
);
410 void LocalFileChangeTracker::RecordChangeToChangeMaps(
411 const FileSystemURL
& url
,
412 const FileChange
& change
,
414 FileChangeMap
* changes
,
415 ChangeSeqMap
* change_seqs
) {
416 ChangeInfo
& info
= (*changes
)[url
];
417 if (info
.change_seq
>= 0 && change_seqs
)
418 change_seqs
->erase(info
.change_seq
);
419 info
.change_list
.Update(change
);
420 if (info
.change_list
.empty()) {
424 info
.change_seq
= new_change_seq
;
426 (*change_seqs
)[info
.change_seq
] = url
;
429 // TrackerDB -------------------------------------------------------------------
431 LocalFileChangeTracker::TrackerDB::TrackerDB(const base::FilePath
& base_path
,
432 leveldb::Env
* env_override
)
433 : base_path_(base_path
),
434 env_override_(env_override
),
435 db_status_(SYNC_STATUS_OK
) {}
437 SyncStatusCode
LocalFileChangeTracker::TrackerDB::Init(
438 RecoveryOption recovery_option
) {
439 if (db_
.get() && db_status_
== SYNC_STATUS_OK
)
440 return SYNC_STATUS_OK
;
442 std::string path
= fileapi::FilePathToString(
443 base_path_
.Append(kDatabaseName
));
444 leveldb::Options options
;
445 options
.max_open_files
= 0; // Use minimum.
446 options
.create_if_missing
= true;
448 options
.env
= env_override_
;
450 leveldb::Status status
= leveldb::DB::Open(options
, path
, &db
);
453 return SYNC_STATUS_OK
;
456 HandleError(FROM_HERE
, status
);
457 if (!status
.IsCorruption())
458 return LevelDBStatusToSyncStatusCode(status
);
460 // Try to repair the corrupted DB.
461 switch (recovery_option
) {
462 case FAIL_ON_CORRUPTION
:
463 return SYNC_DATABASE_ERROR_CORRUPTION
;
464 case REPAIR_ON_CORRUPTION
:
468 return SYNC_DATABASE_ERROR_FAILED
;
471 SyncStatusCode
LocalFileChangeTracker::TrackerDB::Repair(
472 const std::string
& db_path
) {
474 LOG(WARNING
) << "Attempting to repair TrackerDB.";
476 leveldb::Options options
;
477 options
.max_open_files
= 0; // Use minimum.
478 if (leveldb::RepairDB(db_path
, options
).ok() &&
479 Init(FAIL_ON_CORRUPTION
) == SYNC_STATUS_OK
) {
480 // TODO(nhiroki): perform some consistency checks between TrackerDB and
481 // syncable file system.
482 LOG(WARNING
) << "Repairing TrackerDB completed.";
483 return SYNC_STATUS_OK
;
486 LOG(WARNING
) << "Failed to repair TrackerDB.";
487 return SYNC_DATABASE_ERROR_CORRUPTION
;
490 // TODO(nhiroki): factor out the common methods into somewhere else.
491 void LocalFileChangeTracker::TrackerDB::HandleError(
492 const tracked_objects::Location
& from_here
,
493 const leveldb::Status
& status
) {
494 LOG(ERROR
) << "LocalFileChangeTracker::TrackerDB failed at: "
495 << from_here
.ToString() << " with error: " << status
.ToString();
498 SyncStatusCode
LocalFileChangeTracker::TrackerDB::MarkDirty(
499 const std::string
& url
) {
500 if (db_status_
!= SYNC_STATUS_OK
)
503 db_status_
= Init(REPAIR_ON_CORRUPTION
);
504 if (db_status_
!= SYNC_STATUS_OK
) {
509 leveldb::Status status
= db_
->Put(leveldb::WriteOptions(), url
, kMark
);
511 HandleError(FROM_HERE
, status
);
512 db_status_
= LevelDBStatusToSyncStatusCode(status
);
516 return SYNC_STATUS_OK
;
519 SyncStatusCode
LocalFileChangeTracker::TrackerDB::ClearDirty(
520 const std::string
& url
) {
521 if (db_status_
!= SYNC_STATUS_OK
)
524 // Should not reach here before initializing the database. The database should
525 // be cleared after read, and should be initialized during read if
529 leveldb::Status status
= db_
->Delete(leveldb::WriteOptions(), url
);
530 if (!status
.ok() && !status
.IsNotFound()) {
531 HandleError(FROM_HERE
, status
);
532 db_status_
= LevelDBStatusToSyncStatusCode(status
);
536 return SYNC_STATUS_OK
;
539 SyncStatusCode
LocalFileChangeTracker::TrackerDB::GetDirtyEntries(
540 std::queue
<FileSystemURL
>* dirty_files
) {
541 if (db_status_
!= SYNC_STATUS_OK
)
544 db_status_
= Init(REPAIR_ON_CORRUPTION
);
545 if (db_status_
!= SYNC_STATUS_OK
) {
550 scoped_ptr
<leveldb::Iterator
> iter(db_
->NewIterator(leveldb::ReadOptions()));
553 while (iter
->Valid()) {
554 if (!DeserializeSyncableFileSystemURL(iter
->key().ToString(), &url
)) {
555 LOG(WARNING
) << "Failed to deserialize an URL. "
556 << "TrackerDB might be corrupted.";
557 db_status_
= SYNC_DATABASE_ERROR_CORRUPTION
;
561 dirty_files
->push(url
);
564 return SYNC_STATUS_OK
;
567 SyncStatusCode
LocalFileChangeTracker::TrackerDB::WriteBatch(
568 scoped_ptr
<leveldb::WriteBatch
> batch
) {
569 if (db_status_
!= SYNC_STATUS_OK
)
572 leveldb::Status status
= db_
->Write(leveldb::WriteOptions(), batch
.get());
573 if (!status
.ok() && !status
.IsNotFound()) {
574 HandleError(FROM_HERE
, status
);
575 db_status_
= LevelDBStatusToSyncStatusCode(status
);
579 return SYNC_STATUS_OK
;
582 } // namespace sync_file_system