Fire an error if a pref used in the UI is missing once all prefs are fetched.
[chromium-blink-merge.git] / chrome / browser / sync_file_system / local / local_file_change_tracker.cc
blob694b05a97b33696fde67118a99a3a3fa7f354c2a
1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/sync_file_system/local/local_file_change_tracker.h"
7 #include <queue>
9 #include "base/location.h"
10 #include "base/logging.h"
11 #include "base/sequenced_task_runner.h"
12 #include "base/stl_util.h"
13 #include "chrome/browser/sync_file_system/local/local_file_sync_status.h"
14 #include "chrome/browser/sync_file_system/syncable_file_system_util.h"
15 #include "storage/browser/fileapi/file_system_context.h"
16 #include "storage/browser/fileapi/file_system_file_util.h"
17 #include "storage/browser/fileapi/file_system_operation_context.h"
18 #include "storage/common/fileapi/file_system_util.h"
19 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
20 #include "third_party/leveldatabase/src/include/leveldb/db.h"
21 #include "third_party/leveldatabase/src/include/leveldb/env.h"
22 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
24 using storage::FileSystemContext;
25 using storage::FileSystemFileUtil;
26 using storage::FileSystemOperationContext;
27 using storage::FileSystemURL;
28 using storage::FileSystemURLSet;
30 namespace sync_file_system {
32 namespace {
33 const base::FilePath::CharType kDatabaseName[] =
34 FILE_PATH_LITERAL("LocalFileChangeTracker");
35 const char kMark[] = "d";
36 } // namespace
38 // A database class that stores local file changes in a local database. This
39 // object must be destructed on file_task_runner.
40 class LocalFileChangeTracker::TrackerDB {
41 public:
42 TrackerDB(const base::FilePath& base_path,
43 leveldb::Env* env_override);
45 SyncStatusCode MarkDirty(const std::string& url);
46 SyncStatusCode ClearDirty(const std::string& url);
47 SyncStatusCode GetDirtyEntries(
48 std::queue<FileSystemURL>* dirty_files);
49 SyncStatusCode WriteBatch(scoped_ptr<leveldb::WriteBatch> batch);
51 private:
52 enum RecoveryOption {
53 REPAIR_ON_CORRUPTION,
54 FAIL_ON_CORRUPTION,
57 SyncStatusCode Init(RecoveryOption recovery_option);
58 SyncStatusCode Repair(const std::string& db_path);
59 void HandleError(const tracked_objects::Location& from_here,
60 const leveldb::Status& status);
62 const base::FilePath base_path_;
63 leveldb::Env* env_override_;
64 scoped_ptr<leveldb::DB> db_;
65 SyncStatusCode db_status_;
67 DISALLOW_COPY_AND_ASSIGN(TrackerDB);
70 LocalFileChangeTracker::ChangeInfo::ChangeInfo() : change_seq(-1) {}
71 LocalFileChangeTracker::ChangeInfo::~ChangeInfo() {}
73 // LocalFileChangeTracker ------------------------------------------------------
75 LocalFileChangeTracker::LocalFileChangeTracker(
76 const base::FilePath& base_path,
77 leveldb::Env* env_override,
78 base::SequencedTaskRunner* file_task_runner)
79 : initialized_(false),
80 file_task_runner_(file_task_runner),
81 tracker_db_(new TrackerDB(base_path, env_override)),
82 current_change_seq_number_(0),
83 num_changes_(0) {
86 LocalFileChangeTracker::~LocalFileChangeTracker() {
87 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
88 tracker_db_.reset();
91 void LocalFileChangeTracker::OnStartUpdate(const FileSystemURL& url) {
92 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
93 if (ContainsKey(changes_, url) || ContainsKey(demoted_changes_, url))
94 return;
95 // TODO(nhiroki): propagate the error code (see http://crbug.com/152127).
96 MarkDirtyOnDatabase(url);
99 void LocalFileChangeTracker::OnEndUpdate(const FileSystemURL& url) {}
101 void LocalFileChangeTracker::OnCreateFile(const FileSystemURL& url) {
102 RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
103 SYNC_FILE_TYPE_FILE));
106 void LocalFileChangeTracker::OnCreateFileFrom(const FileSystemURL& url,
107 const FileSystemURL& src) {
108 RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
109 SYNC_FILE_TYPE_FILE));
112 void LocalFileChangeTracker::OnRemoveFile(const FileSystemURL& url) {
113 RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE,
114 SYNC_FILE_TYPE_FILE));
117 void LocalFileChangeTracker::OnModifyFile(const FileSystemURL& url) {
118 RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
119 SYNC_FILE_TYPE_FILE));
122 void LocalFileChangeTracker::OnCreateDirectory(const FileSystemURL& url) {
123 RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
124 SYNC_FILE_TYPE_DIRECTORY));
127 void LocalFileChangeTracker::OnRemoveDirectory(const FileSystemURL& url) {
128 RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE,
129 SYNC_FILE_TYPE_DIRECTORY));
132 void LocalFileChangeTracker::GetNextChangedURLs(
133 std::deque<FileSystemURL>* urls, int max_urls) {
134 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
135 DCHECK(urls);
136 urls->clear();
137 // Mildly prioritizes the URLs that older changes and have not been updated
138 // for a while.
139 for (ChangeSeqMap::iterator iter = change_seqs_.begin();
140 iter != change_seqs_.end() &&
141 (max_urls == 0 || urls->size() < static_cast<size_t>(max_urls));
142 ++iter) {
143 urls->push_back(iter->second);
147 void LocalFileChangeTracker::GetChangesForURL(
148 const FileSystemURL& url, FileChangeList* changes) {
149 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
150 DCHECK(changes);
151 changes->clear();
152 FileChangeMap::iterator found = changes_.find(url);
153 if (found == changes_.end()) {
154 found = demoted_changes_.find(url);
155 if (found == demoted_changes_.end())
156 return;
158 *changes = found->second.change_list;
161 void LocalFileChangeTracker::ClearChangesForURL(const FileSystemURL& url) {
162 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
163 ClearDirtyOnDatabase(url);
164 mirror_changes_.erase(url);
165 demoted_changes_.erase(url);
166 FileChangeMap::iterator found = changes_.find(url);
167 if (found == changes_.end())
168 return;
169 change_seqs_.erase(found->second.change_seq);
170 changes_.erase(found);
171 UpdateNumChanges();
174 void LocalFileChangeTracker::CreateFreshMirrorForURL(
175 const storage::FileSystemURL& url) {
176 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
177 DCHECK(!ContainsKey(mirror_changes_, url));
178 mirror_changes_[url] = ChangeInfo();
181 void LocalFileChangeTracker::RemoveMirrorAndCommitChangesForURL(
182 const storage::FileSystemURL& url) {
183 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
184 FileChangeMap::iterator found = mirror_changes_.find(url);
185 if (found == mirror_changes_.end())
186 return;
187 mirror_changes_.erase(found);
189 if (ContainsKey(changes_, url) || ContainsKey(demoted_changes_, url))
190 MarkDirtyOnDatabase(url);
191 else
192 ClearDirtyOnDatabase(url);
193 UpdateNumChanges();
196 void LocalFileChangeTracker::ResetToMirrorAndCommitChangesForURL(
197 const storage::FileSystemURL& url) {
198 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
199 FileChangeMap::iterator found = mirror_changes_.find(url);
200 if (found == mirror_changes_.end() || found->second.change_list.empty()) {
201 ClearChangesForURL(url);
202 return;
204 const ChangeInfo& info = found->second;
205 if (ContainsKey(demoted_changes_, url)) {
206 DCHECK(!ContainsKey(changes_, url));
207 demoted_changes_[url] = info;
208 } else {
209 DCHECK(!ContainsKey(demoted_changes_, url));
210 change_seqs_[info.change_seq] = url;
211 changes_[url] = info;
213 RemoveMirrorAndCommitChangesForURL(url);
216 void LocalFileChangeTracker::DemoteChangesForURL(
217 const storage::FileSystemURL& url) {
218 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
220 FileChangeMap::iterator found = changes_.find(url);
221 if (found == changes_.end())
222 return;
223 DCHECK(!ContainsKey(demoted_changes_, url));
224 change_seqs_.erase(found->second.change_seq);
225 demoted_changes_.insert(*found);
226 changes_.erase(found);
227 UpdateNumChanges();
230 void LocalFileChangeTracker::PromoteDemotedChangesForURL(
231 const storage::FileSystemURL& url) {
232 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
234 FileChangeMap::iterator iter = demoted_changes_.find(url);
235 if (iter == demoted_changes_.end())
236 return;
238 FileChangeList::List change_list = iter->second.change_list.list();
239 // Make sure that this URL is in no queues.
240 DCHECK(!ContainsKey(change_seqs_, iter->second.change_seq));
241 DCHECK(!ContainsKey(changes_, url));
243 change_seqs_[iter->second.change_seq] = url;
244 changes_.insert(*iter);
245 demoted_changes_.erase(iter);
246 UpdateNumChanges();
249 bool LocalFileChangeTracker::PromoteDemotedChanges() {
250 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
251 if (demoted_changes_.empty())
252 return false;
253 while (!demoted_changes_.empty()) {
254 storage::FileSystemURL url = demoted_changes_.begin()->first;
255 PromoteDemotedChangesForURL(url);
257 UpdateNumChanges();
258 return true;
261 SyncStatusCode LocalFileChangeTracker::Initialize(
262 FileSystemContext* file_system_context) {
263 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
264 DCHECK(!initialized_);
265 DCHECK(file_system_context);
267 SyncStatusCode status = CollectLastDirtyChanges(file_system_context);
268 if (status == SYNC_STATUS_OK)
269 initialized_ = true;
270 return status;
273 void LocalFileChangeTracker::ResetForFileSystem(const GURL& origin,
274 storage::FileSystemType type) {
275 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
276 scoped_ptr<leveldb::WriteBatch> batch(new leveldb::WriteBatch);
277 for (FileChangeMap::iterator iter = changes_.begin();
278 iter != changes_.end();) {
279 storage::FileSystemURL url = iter->first;
280 int change_seq = iter->second.change_seq;
281 // Advance |iter| before calling ResetForURL to avoid the iterator
282 // invalidation in it.
283 ++iter;
284 if (url.origin() == origin && url.type() == type)
285 ResetForURL(url, change_seq, batch.get());
288 for (FileChangeMap::iterator iter = demoted_changes_.begin();
289 iter != demoted_changes_.end();) {
290 storage::FileSystemURL url = iter->first;
291 int change_seq = iter->second.change_seq;
292 // Advance |iter| before calling ResetForURL to avoid the iterator
293 // invalidation in it.
294 ++iter;
295 if (url.origin() == origin && url.type() == type)
296 ResetForURL(url, change_seq, batch.get());
299 // Fail to apply batch to database wouldn't have critical effect, they'll be
300 // just marked deleted on next relaunch.
301 tracker_db_->WriteBatch(batch.Pass());
302 UpdateNumChanges();
305 void LocalFileChangeTracker::UpdateNumChanges() {
306 base::AutoLock lock(num_changes_lock_);
307 num_changes_ = static_cast<int64>(change_seqs_.size());
310 void LocalFileChangeTracker::GetAllChangedURLs(FileSystemURLSet* urls) {
311 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
312 std::deque<FileSystemURL> url_deque;
313 GetNextChangedURLs(&url_deque, 0);
314 urls->clear();
315 urls->insert(url_deque.begin(), url_deque.end());
318 void LocalFileChangeTracker::DropAllChanges() {
319 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
320 changes_.clear();
321 change_seqs_.clear();
322 mirror_changes_.clear();
323 UpdateNumChanges();
326 SyncStatusCode LocalFileChangeTracker::MarkDirtyOnDatabase(
327 const FileSystemURL& url) {
328 std::string serialized_url;
329 if (!SerializeSyncableFileSystemURL(url, &serialized_url))
330 return SYNC_FILE_ERROR_INVALID_URL;
332 return tracker_db_->MarkDirty(serialized_url);
335 SyncStatusCode LocalFileChangeTracker::ClearDirtyOnDatabase(
336 const FileSystemURL& url) {
337 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
338 std::string serialized_url;
339 if (!SerializeSyncableFileSystemURL(url, &serialized_url))
340 return SYNC_FILE_ERROR_INVALID_URL;
342 return tracker_db_->ClearDirty(serialized_url);
345 SyncStatusCode LocalFileChangeTracker::CollectLastDirtyChanges(
346 FileSystemContext* file_system_context) {
347 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
349 std::queue<FileSystemURL> dirty_files;
350 const SyncStatusCode status = tracker_db_->GetDirtyEntries(&dirty_files);
351 if (status != SYNC_STATUS_OK)
352 return status;
354 FileSystemFileUtil* file_util =
355 file_system_context->sandbox_delegate()->sync_file_util();
356 DCHECK(file_util);
357 scoped_ptr<FileSystemOperationContext> context(
358 new FileSystemOperationContext(file_system_context));
360 base::File::Info file_info;
361 base::FilePath platform_path;
363 while (!dirty_files.empty()) {
364 const FileSystemURL url = dirty_files.front();
365 dirty_files.pop();
366 DCHECK_EQ(url.type(), storage::kFileSystemTypeSyncable);
368 switch (file_util->GetFileInfo(context.get(), url,
369 &file_info, &platform_path)) {
370 case base::File::FILE_OK: {
371 if (!file_info.is_directory) {
372 RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
373 SYNC_FILE_TYPE_FILE));
374 break;
377 RecordChange(url, FileChange(
378 FileChange::FILE_CHANGE_ADD_OR_UPDATE,
379 SYNC_FILE_TYPE_DIRECTORY));
381 // Push files and directories in this directory into |dirty_files|.
382 scoped_ptr<FileSystemFileUtil::AbstractFileEnumerator> enumerator(
383 file_util->CreateFileEnumerator(context.get(), url));
384 base::FilePath path_each;
385 while (!(path_each = enumerator->Next()).empty()) {
386 dirty_files.push(CreateSyncableFileSystemURL(
387 url.origin(), path_each));
389 break;
391 case base::File::FILE_ERROR_NOT_FOUND: {
392 // File represented by |url| has already been deleted. Since we cannot
393 // figure out if this file was directory or not from the URL, file
394 // type is treated as SYNC_FILE_TYPE_UNKNOWN.
396 // NOTE: Directory to have been reverted (that is, ADD -> DELETE) is
397 // also treated as FILE_CHANGE_DELETE.
398 RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE,
399 SYNC_FILE_TYPE_UNKNOWN));
400 break;
402 case base::File::FILE_ERROR_FAILED:
403 default:
404 // TODO(nhiroki): handle file access error (http://crbug.com/155251).
405 LOG(WARNING) << "Failed to access local file.";
406 break;
409 return SYNC_STATUS_OK;
412 void LocalFileChangeTracker::RecordChange(
413 const FileSystemURL& url, const FileChange& change) {
414 DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
415 int change_seq = current_change_seq_number_++;
416 if (ContainsKey(demoted_changes_, url)) {
417 RecordChangeToChangeMaps(url, change, change_seq,
418 &demoted_changes_, nullptr);
419 } else {
420 RecordChangeToChangeMaps(url, change, change_seq, &changes_, &change_seqs_);
422 if (ContainsKey(mirror_changes_, url)) {
423 RecordChangeToChangeMaps(url, change, change_seq, &mirror_changes_,
424 nullptr);
426 UpdateNumChanges();
429 // static
430 void LocalFileChangeTracker::RecordChangeToChangeMaps(
431 const FileSystemURL& url,
432 const FileChange& change,
433 int new_change_seq,
434 FileChangeMap* changes,
435 ChangeSeqMap* change_seqs) {
436 ChangeInfo& info = (*changes)[url];
437 if (info.change_seq >= 0 && change_seqs)
438 change_seqs->erase(info.change_seq);
439 info.change_list.Update(change);
440 if (info.change_list.empty()) {
441 changes->erase(url);
442 return;
444 info.change_seq = new_change_seq;
445 if (change_seqs)
446 (*change_seqs)[info.change_seq] = url;
449 void LocalFileChangeTracker::ResetForURL(const storage::FileSystemURL& url,
450 int change_seq,
451 leveldb::WriteBatch* batch) {
452 mirror_changes_.erase(url);
453 demoted_changes_.erase(url);
454 change_seqs_.erase(change_seq);
455 changes_.erase(url);
457 std::string serialized_url;
458 if (!SerializeSyncableFileSystemURL(url, &serialized_url)) {
459 NOTREACHED() << "Failed to serialize: " << url.DebugString();
460 return;
462 batch->Delete(serialized_url);
465 // TrackerDB -------------------------------------------------------------------
467 LocalFileChangeTracker::TrackerDB::TrackerDB(const base::FilePath& base_path,
468 leveldb::Env* env_override)
469 : base_path_(base_path),
470 env_override_(env_override),
471 db_status_(SYNC_STATUS_OK) {}
473 SyncStatusCode LocalFileChangeTracker::TrackerDB::Init(
474 RecoveryOption recovery_option) {
475 if (db_.get() && db_status_ == SYNC_STATUS_OK)
476 return SYNC_STATUS_OK;
478 std::string path =
479 storage::FilePathToString(base_path_.Append(kDatabaseName));
480 leveldb::Options options;
481 options.max_open_files = 0; // Use minimum.
482 options.create_if_missing = true;
483 if (env_override_)
484 options.env = env_override_;
485 leveldb::DB* db;
486 leveldb::Status status = leveldb::DB::Open(options, path, &db);
487 if (status.ok()) {
488 db_.reset(db);
489 return SYNC_STATUS_OK;
492 HandleError(FROM_HERE, status);
493 if (!status.IsCorruption())
494 return LevelDBStatusToSyncStatusCode(status);
496 // Try to repair the corrupted DB.
497 switch (recovery_option) {
498 case FAIL_ON_CORRUPTION:
499 return SYNC_DATABASE_ERROR_CORRUPTION;
500 case REPAIR_ON_CORRUPTION:
501 return Repair(path);
503 NOTREACHED();
504 return SYNC_DATABASE_ERROR_FAILED;
507 SyncStatusCode LocalFileChangeTracker::TrackerDB::Repair(
508 const std::string& db_path) {
509 DCHECK(!db_.get());
510 LOG(WARNING) << "Attempting to repair TrackerDB.";
512 leveldb::Options options;
513 options.max_open_files = 0; // Use minimum.
514 if (leveldb::RepairDB(db_path, options).ok() &&
515 Init(FAIL_ON_CORRUPTION) == SYNC_STATUS_OK) {
516 // TODO(nhiroki): perform some consistency checks between TrackerDB and
517 // syncable file system.
518 LOG(WARNING) << "Repairing TrackerDB completed.";
519 return SYNC_STATUS_OK;
522 LOG(WARNING) << "Failed to repair TrackerDB.";
523 return SYNC_DATABASE_ERROR_CORRUPTION;
526 // TODO(nhiroki): factor out the common methods into somewhere else.
527 void LocalFileChangeTracker::TrackerDB::HandleError(
528 const tracked_objects::Location& from_here,
529 const leveldb::Status& status) {
530 LOG(ERROR) << "LocalFileChangeTracker::TrackerDB failed at: "
531 << from_here.ToString() << " with error: " << status.ToString();
534 SyncStatusCode LocalFileChangeTracker::TrackerDB::MarkDirty(
535 const std::string& url) {
536 if (db_status_ != SYNC_STATUS_OK)
537 return db_status_;
539 db_status_ = Init(REPAIR_ON_CORRUPTION);
540 if (db_status_ != SYNC_STATUS_OK) {
541 db_.reset();
542 return db_status_;
545 leveldb::Status status = db_->Put(leveldb::WriteOptions(), url, kMark);
546 if (!status.ok()) {
547 HandleError(FROM_HERE, status);
548 db_status_ = LevelDBStatusToSyncStatusCode(status);
549 db_.reset();
550 return db_status_;
552 return SYNC_STATUS_OK;
555 SyncStatusCode LocalFileChangeTracker::TrackerDB::ClearDirty(
556 const std::string& url) {
557 if (db_status_ != SYNC_STATUS_OK)
558 return db_status_;
560 // Should not reach here before initializing the database. The database should
561 // be cleared after read, and should be initialized during read if
562 // uninitialized.
563 DCHECK(db_.get());
565 leveldb::Status status = db_->Delete(leveldb::WriteOptions(), url);
566 if (!status.ok() && !status.IsNotFound()) {
567 HandleError(FROM_HERE, status);
568 db_status_ = LevelDBStatusToSyncStatusCode(status);
569 db_.reset();
570 return db_status_;
572 return SYNC_STATUS_OK;
575 SyncStatusCode LocalFileChangeTracker::TrackerDB::GetDirtyEntries(
576 std::queue<FileSystemURL>* dirty_files) {
577 if (db_status_ != SYNC_STATUS_OK)
578 return db_status_;
580 db_status_ = Init(REPAIR_ON_CORRUPTION);
581 if (db_status_ != SYNC_STATUS_OK) {
582 db_.reset();
583 return db_status_;
586 scoped_ptr<leveldb::Iterator> iter(db_->NewIterator(leveldb::ReadOptions()));
587 iter->SeekToFirst();
588 FileSystemURL url;
589 while (iter->Valid()) {
590 if (!DeserializeSyncableFileSystemURL(iter->key().ToString(), &url)) {
591 LOG(WARNING) << "Failed to deserialize an URL. "
592 << "TrackerDB might be corrupted.";
593 db_status_ = SYNC_DATABASE_ERROR_CORRUPTION;
594 db_.reset();
595 return db_status_;
597 dirty_files->push(url);
598 iter->Next();
600 return SYNC_STATUS_OK;
603 SyncStatusCode LocalFileChangeTracker::TrackerDB::WriteBatch(
604 scoped_ptr<leveldb::WriteBatch> batch) {
605 if (db_status_ != SYNC_STATUS_OK)
606 return db_status_;
608 leveldb::Status status = db_->Write(leveldb::WriteOptions(), batch.get());
609 if (!status.ok() && !status.IsNotFound()) {
610 HandleError(FROM_HERE, status);
611 db_status_ = LevelDBStatusToSyncStatusCode(status);
612 db_.reset();
613 return db_status_;
615 return SYNC_STATUS_OK;
618 } // namespace sync_file_system