chrome/browser/sync_file_system/local/local_file_change_tracker.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/sync_file_system/local/local_file_change_tracker.h"
   6
   7 #include <queue>
   8
   9 #include "base/location.h"
  10 #include "base/logging.h"
  11 #include "base/sequenced_task_runner.h"
  12 #include "base/stl_util.h"
  13 #include "chrome/browser/sync_file_system/local/local_file_sync_status.h"
  14 #include "chrome/browser/sync_file_system/syncable_file_system_util.h"
  15 #include "third_party/leveldatabase/src/include/leveldb/db.h"
  16 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
  17 #include "webkit/browser/fileapi/file_system_context.h"
  18 #include "webkit/browser/fileapi/file_system_file_util.h"
  19 #include "webkit/browser/fileapi/file_system_operation_context.h"
  20 #include "webkit/common/fileapi/file_system_util.h"
  21
  22 using fileapi::FileSystemContext;
  23 using fileapi::FileSystemFileUtil;
  24 using fileapi::FileSystemOperationContext;
  25 using fileapi::FileSystemURL;
  26 using fileapi::FileSystemURLSet;
  27
  28 namespace sync_file_system {
  29
  30 namespace {
  31 const base::FilePath::CharType kDatabaseName[] =
  32     FILE_PATH_LITERAL("LocalFileChangeTracker");
  33 const char kMark[] = "d";
  34 }  // namespace
  35
  36 // A database class that stores local file changes in a local database. This
  37 // object must be destructed on file_task_runner.
  38 class LocalFileChangeTracker::TrackerDB {
  39  public:
  40   explicit TrackerDB(const base::FilePath& base_path);
  41
  42   SyncStatusCode MarkDirty(const std::string& url);
  43   SyncStatusCode ClearDirty(const std::string& url);
  44   SyncStatusCode GetDirtyEntries(
  45       std::queue<FileSystemURL>* dirty_files);
  46   SyncStatusCode WriteBatch(scoped_ptr<leveldb::WriteBatch> batch);
  47
  48  private:
  49   enum RecoveryOption {
  50     REPAIR_ON_CORRUPTION,
  51     FAIL_ON_CORRUPTION,
  52   };
  53
  54   SyncStatusCode Init(RecoveryOption recovery_option);
  55   SyncStatusCode Repair(const std::string& db_path);
  56   void HandleError(const tracked_objects::Location& from_here,
  57                    const leveldb::Status& status);
  58
  59   const base::FilePath base_path_;
  60   scoped_ptr<leveldb::DB> db_;
  61   SyncStatusCode db_status_;
  62
  63   DISALLOW_COPY_AND_ASSIGN(TrackerDB);
  64 };
  65
  66 LocalFileChangeTracker::ChangeInfo::ChangeInfo() : change_seq(-1) {}
  67 LocalFileChangeTracker::ChangeInfo::~ChangeInfo() {}
  68
  69 // LocalFileChangeTracker ------------------------------------------------------
  70
  71 LocalFileChangeTracker::LocalFileChangeTracker(
  72     const base::FilePath& base_path,
  73     base::SequencedTaskRunner* file_task_runner)
  74     : initialized_(false),
  75       file_task_runner_(file_task_runner),
  76       tracker_db_(new TrackerDB(base_path)),
  77       current_change_seq_(0),
  78       num_changes_(0) {
  79 }
  80
  81 LocalFileChangeTracker::~LocalFileChangeTracker() {
  82   DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
  83   tracker_db_.reset();
  84 }
  85
  86 void LocalFileChangeTracker::OnStartUpdate(const FileSystemURL& url) {
  87   DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
  88   if (ContainsKey(changes_, url))
  89     return;
  90   // TODO(nhiroki): propagate the error code (see http://crbug.com/152127).
  91   MarkDirtyOnDatabase(url);
  92 }
  93
  94 void LocalFileChangeTracker::OnEndUpdate(const FileSystemURL& url) {}
  95
  96 void LocalFileChangeTracker::OnCreateFile(const FileSystemURL& url) {
  97   RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
  98                                SYNC_FILE_TYPE_FILE));
  99 }
 100
 101 void LocalFileChangeTracker::OnCreateFileFrom(const FileSystemURL& url,
 102                                               const FileSystemURL& src) {
 103   RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
 104                                SYNC_FILE_TYPE_FILE));
 105 }
 106
 107 void LocalFileChangeTracker::OnRemoveFile(const FileSystemURL& url) {
 108   RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE,
 109                                SYNC_FILE_TYPE_FILE));
 110 }
 111
 112 void LocalFileChangeTracker::OnModifyFile(const FileSystemURL& url) {
 113   RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
 114                                SYNC_FILE_TYPE_FILE));
 115 }
 116
 117 void LocalFileChangeTracker::OnCreateDirectory(const FileSystemURL& url) {
 118   RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
 119                                SYNC_FILE_TYPE_DIRECTORY));
 120 }
 121
 122 void LocalFileChangeTracker::OnRemoveDirectory(const FileSystemURL& url) {
 123   RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE,
 124                                SYNC_FILE_TYPE_DIRECTORY));
 125 }
 126
 127 void LocalFileChangeTracker::GetNextChangedURLs(
 128     std::deque<FileSystemURL>* urls, int max_urls) {
 129   DCHECK(urls);
 130   DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
 131   urls->clear();
 132   // Mildly prioritizes the URLs that older changes and have not been updated
 133   // for a while.
 134   for (ChangeSeqMap::iterator iter = change_seqs_.begin();
 135        iter != change_seqs_.end() &&
 136        (max_urls == 0 || urls->size() < static_cast<size_t>(max_urls));
 137        ++iter) {
 138     urls->push_back(iter->second);
 139   }
 140 }
 141
 142 void LocalFileChangeTracker::GetChangesForURL(
 143     const FileSystemURL& url, FileChangeList* changes) {
 144   DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
 145   DCHECK(changes);
 146   changes->clear();
 147   FileChangeMap::iterator found = changes_.find(url);
 148   if (found == changes_.end()) {
 149     found = demoted_changes_.find(url);
 150     if (found == demoted_changes_.end())
 151       return;
 152   }
 153   *changes = found->second.change_list;
 154 }
 155
 156 void LocalFileChangeTracker::ClearChangesForURL(const FileSystemURL& url) {
 157   DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
 158   ClearDirtyOnDatabase(url);
 159   mirror_changes_.erase(url);
 160   demoted_changes_.erase(url);
 161   FileChangeMap::iterator found = changes_.find(url);
 162   if (found == changes_.end())
 163     return;
 164   change_seqs_.erase(found->second.change_seq);
 165   changes_.erase(found);
 166   UpdateNumChanges();
 167 }
 168
 169 void LocalFileChangeTracker::CreateFreshMirrorForURL(
 170     const fileapi::FileSystemURL& url) {
 171   DCHECK(!ContainsKey(mirror_changes_, url));
 172   mirror_changes_[url] = ChangeInfo();
 173 }
 174
 175 void LocalFileChangeTracker::RemoveMirrorAndCommitChangesForURL(
 176     const fileapi::FileSystemURL& url) {
 177   FileChangeMap::iterator found = mirror_changes_.find(url);
 178   if (found == mirror_changes_.end())
 179     return;
 180   mirror_changes_.erase(found);
 181
 182   if (ContainsKey(changes_, url))
 183     MarkDirtyOnDatabase(url);
 184   else
 185     ClearDirtyOnDatabase(url);
 186   UpdateNumChanges();
 187 }
 188
 189 void LocalFileChangeTracker::ResetToMirrorAndCommitChangesForURL(
 190     const fileapi::FileSystemURL& url) {
 191   FileChangeMap::iterator found = mirror_changes_.find(url);
 192   if (found == mirror_changes_.end() || found->second.change_list.empty()) {
 193     ClearChangesForURL(url);
 194     return;
 195   }
 196   const ChangeInfo& info = found->second;
 197   change_seqs_[info.change_seq] = url;
 198   changes_[url] = info;
 199   RemoveMirrorAndCommitChangesForURL(url);
 200 }
 201
 202 void LocalFileChangeTracker::DemoteChangesForURL(
 203     const fileapi::FileSystemURL& url) {
 204   FileChangeMap::iterator found = changes_.find(url);
 205   if (found == changes_.end())
 206     return;
 207   FileChangeList changes = found->second.change_list;
 208
 209   mirror_changes_.erase(url);
 210   change_seqs_.erase(found->second.change_seq);
 211   changes_.erase(found);
 212
 213   FileChangeList::List change_list = changes.list();
 214   while (!change_list.empty()) {
 215     RecordChangeToChangeMaps(url, change_list.front(), 0,
 216                              &demoted_changes_, NULL);
 217     change_list.pop_front();
 218   }
 219 }
 220
 221 bool LocalFileChangeTracker::PromoteDemotedChanges() {
 222   if (demoted_changes_.empty())
 223     return false;
 224   for (FileChangeMap::iterator iter = demoted_changes_.begin();
 225        iter != demoted_changes_.end();) {
 226     fileapi::FileSystemURL url = iter->first;
 227     FileChangeList::List change_list = iter->second.change_list.list();
 228     demoted_changes_.erase(iter++);
 229
 230     // Make sure that this URL is in no queues.
 231     DCHECK(!ContainsKey(changes_, url));
 232     DCHECK(!ContainsKey(demoted_changes_, url));
 233     DCHECK(!ContainsKey(mirror_changes_, url));
 234
 235     while (!change_list.empty()) {
 236       RecordChange(url, change_list.front());
 237       change_list.pop_front();
 238     }
 239   }
 240   demoted_changes_.clear();
 241   return true;
 242 }
 243
 244 SyncStatusCode LocalFileChangeTracker::Initialize(
 245     FileSystemContext* file_system_context) {
 246   DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
 247   DCHECK(!initialized_);
 248   DCHECK(file_system_context);
 249
 250   SyncStatusCode status = CollectLastDirtyChanges(file_system_context);
 251   if (status == SYNC_STATUS_OK)
 252     initialized_ = true;
 253   return status;
 254 }
 255
 256 void LocalFileChangeTracker::ResetForFileSystem(
 257     const GURL& origin,
 258     fileapi::FileSystemType type) {
 259   DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
 260   scoped_ptr<leveldb::WriteBatch> batch(new leveldb::WriteBatch);
 261   for (FileChangeMap::iterator iter = changes_.begin();
 262        iter != changes_.end();) {
 263     fileapi::FileSystemURL url = iter->first;
 264     if (url.origin() != origin || url.type() != type) {
 265       ++iter;
 266       continue;
 267     }
 268     mirror_changes_.erase(url);
 269     demoted_changes_.erase(url);
 270     change_seqs_.erase(iter->second.change_seq);
 271     changes_.erase(iter++);
 272
 273     std::string serialized_url;
 274     const bool should_success =
 275         SerializeSyncableFileSystemURL(url, &serialized_url);
 276     if (!should_success) {
 277       NOTREACHED() << "Failed to serialize: " << url.DebugString();
 278       continue;
 279     }
 280     batch->Delete(serialized_url);
 281   }
 282   // Fail to apply batch to database wouldn't have critical effect, they'll be
 283   // just marked deleted on next relaunch.
 284   tracker_db_->WriteBatch(batch.Pass());
 285   UpdateNumChanges();
 286 }
 287
 288 void LocalFileChangeTracker::UpdateNumChanges() {
 289   base::AutoLock lock(num_changes_lock_);
 290   num_changes_ = static_cast<int64>(change_seqs_.size());
 291 }
 292
 293 void LocalFileChangeTracker::GetAllChangedURLs(FileSystemURLSet* urls) {
 294   std::deque<FileSystemURL> url_deque;
 295   GetNextChangedURLs(&url_deque, 0);
 296   urls->clear();
 297   urls->insert(url_deque.begin(), url_deque.end());
 298 }
 299
 300 void LocalFileChangeTracker::DropAllChanges() {
 301   changes_.clear();
 302   change_seqs_.clear();
 303   mirror_changes_.clear();
 304 }
 305
 306 SyncStatusCode LocalFileChangeTracker::MarkDirtyOnDatabase(
 307     const FileSystemURL& url) {
 308   std::string serialized_url;
 309   if (!SerializeSyncableFileSystemURL(url, &serialized_url))
 310     return SYNC_FILE_ERROR_INVALID_URL;
 311
 312   return tracker_db_->MarkDirty(serialized_url);
 313 }
 314
 315 SyncStatusCode LocalFileChangeTracker::ClearDirtyOnDatabase(
 316     const FileSystemURL& url) {
 317   std::string serialized_url;
 318   if (!SerializeSyncableFileSystemURL(url, &serialized_url))
 319     return SYNC_FILE_ERROR_INVALID_URL;
 320
 321   return tracker_db_->ClearDirty(serialized_url);
 322 }
 323
 324 SyncStatusCode LocalFileChangeTracker::CollectLastDirtyChanges(
 325     FileSystemContext* file_system_context) {
 326   DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
 327
 328   std::queue<FileSystemURL> dirty_files;
 329   const SyncStatusCode status = tracker_db_->GetDirtyEntries(&dirty_files);
 330   if (status != SYNC_STATUS_OK)
 331     return status;
 332
 333   FileSystemFileUtil* file_util =
 334       file_system_context->sandbox_delegate()->sync_file_util();
 335   DCHECK(file_util);
 336   scoped_ptr<FileSystemOperationContext> context(
 337       new FileSystemOperationContext(file_system_context));
 338
 339   base::PlatformFileInfo file_info;
 340   base::FilePath platform_path;
 341
 342   while (!dirty_files.empty()) {
 343     const FileSystemURL url = dirty_files.front();
 344     dirty_files.pop();
 345     DCHECK_EQ(url.type(), fileapi::kFileSystemTypeSyncable);
 346
 347     switch (file_util->GetFileInfo(context.get(), url,
 348                                    &file_info, &platform_path)) {
 349       case base::PLATFORM_FILE_OK: {
 350         if (!file_info.is_directory) {
 351           RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
 352                                        SYNC_FILE_TYPE_FILE));
 353           break;
 354         }
 355
 356         RecordChange(url, FileChange(
 357             FileChange::FILE_CHANGE_ADD_OR_UPDATE,
 358             SYNC_FILE_TYPE_DIRECTORY));
 359
 360         // Push files and directories in this directory into |dirty_files|.
 361         scoped_ptr<FileSystemFileUtil::AbstractFileEnumerator> enumerator(
 362             file_util->CreateFileEnumerator(context.get(), url));
 363         base::FilePath path_each;
 364         while (!(path_each = enumerator->Next()).empty()) {
 365           dirty_files.push(CreateSyncableFileSystemURL(
 366                   url.origin(), path_each));
 367         }
 368         break;
 369       }
 370       case base::PLATFORM_FILE_ERROR_NOT_FOUND: {
 371         // File represented by |url| has already been deleted. Since we cannot
 372         // figure out if this file was directory or not from the URL, file
 373         // type is treated as SYNC_FILE_TYPE_UNKNOWN.
 374         //
 375         // NOTE: Directory to have been reverted (that is, ADD -> DELETE) is
 376         // also treated as FILE_CHANGE_DELETE.
 377         RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE,
 378                                      SYNC_FILE_TYPE_UNKNOWN));
 379         break;
 380       }
 381       case base::PLATFORM_FILE_ERROR_FAILED:
 382       default:
 383         // TODO(nhiroki): handle file access error (http://crbug.com/155251).
 384         LOG(WARNING) << "Failed to access local file.";
 385         break;
 386     }
 387   }
 388   return SYNC_STATUS_OK;
 389 }
 390
 391 void LocalFileChangeTracker::RecordChange(
 392     const FileSystemURL& url, const FileChange& change) {
 393   DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
 394   if (ContainsKey(demoted_changes_, url)) {
 395     RecordChangeToChangeMaps(url, change, 0, &demoted_changes_, NULL);
 396     return;
 397   }
 398   int change_seq = current_change_seq_++;
 399   RecordChangeToChangeMaps(url, change, change_seq, &changes_, &change_seqs_);
 400   if (ContainsKey(mirror_changes_, url))
 401     RecordChangeToChangeMaps(url, change, change_seq, &mirror_changes_, NULL);
 402   UpdateNumChanges();
 403 }
 404
 405 void LocalFileChangeTracker::RecordChangeToChangeMaps(
 406     const FileSystemURL& url,
 407     const FileChange& change,
 408     int new_change_seq,
 409     FileChangeMap* changes,
 410     ChangeSeqMap* change_seqs) {
 411   ChangeInfo& info = (*changes)[url];
 412   if (info.change_seq >= 0 && change_seqs)
 413     change_seqs->erase(info.change_seq);
 414   info.change_list.Update(change);
 415   if (info.change_list.empty()) {
 416     changes->erase(url);
 417     return;
 418   }
 419   info.change_seq = new_change_seq;
 420   if (change_seqs)
 421     (*change_seqs)[info.change_seq] = url;
 422 }
 423
 424 // TrackerDB -------------------------------------------------------------------
 425
 426 LocalFileChangeTracker::TrackerDB::TrackerDB(const base::FilePath& base_path)
 427   : base_path_(base_path),
 428     db_status_(SYNC_STATUS_OK) {}
 429
 430 SyncStatusCode LocalFileChangeTracker::TrackerDB::Init(
 431     RecoveryOption recovery_option) {
 432   if (db_.get() && db_status_ == SYNC_STATUS_OK)
 433     return SYNC_STATUS_OK;
 434
 435   std::string path = fileapi::FilePathToString(
 436       base_path_.Append(kDatabaseName));
 437   leveldb::Options options;
 438   options.max_open_files = 0;  // Use minimum.
 439   options.create_if_missing = true;
 440   leveldb::DB* db;
 441   leveldb::Status status = leveldb::DB::Open(options, path, &db);
 442   if (status.ok()) {
 443     db_.reset(db);
 444     return SYNC_STATUS_OK;
 445   }
 446
 447   HandleError(FROM_HERE, status);
 448   if (!status.IsCorruption())
 449     return LevelDBStatusToSyncStatusCode(status);
 450
 451   // Try to repair the corrupted DB.
 452   switch (recovery_option) {
 453     case FAIL_ON_CORRUPTION:
 454       return SYNC_DATABASE_ERROR_CORRUPTION;
 455     case REPAIR_ON_CORRUPTION:
 456       return Repair(path);
 457   }
 458   NOTREACHED();
 459   return SYNC_DATABASE_ERROR_FAILED;
 460 }
 461
 462 SyncStatusCode LocalFileChangeTracker::TrackerDB::Repair(
 463     const std::string& db_path) {
 464   DCHECK(!db_.get());
 465   LOG(WARNING) << "Attempting to repair TrackerDB.";
 466
 467   leveldb::Options options;
 468   options.max_open_files = 0;  // Use minimum.
 469   if (leveldb::RepairDB(db_path, options).ok() &&
 470       Init(FAIL_ON_CORRUPTION) == SYNC_STATUS_OK) {
 471     // TODO(nhiroki): perform some consistency checks between TrackerDB and
 472     // syncable file system.
 473     LOG(WARNING) << "Repairing TrackerDB completed.";
 474     return SYNC_STATUS_OK;
 475   }
 476
 477   LOG(WARNING) << "Failed to repair TrackerDB.";
 478   return SYNC_DATABASE_ERROR_CORRUPTION;
 479 }
 480
 481 // TODO(nhiroki): factor out the common methods into somewhere else.
 482 void LocalFileChangeTracker::TrackerDB::HandleError(
 483     const tracked_objects::Location& from_here,
 484     const leveldb::Status& status) {
 485   LOG(ERROR) << "LocalFileChangeTracker::TrackerDB failed at: "
 486              << from_here.ToString() << " with error: " << status.ToString();
 487 }
 488
 489 SyncStatusCode LocalFileChangeTracker::TrackerDB::MarkDirty(
 490     const std::string& url) {
 491   if (db_status_ != SYNC_STATUS_OK)
 492     return db_status_;
 493
 494   db_status_ = Init(REPAIR_ON_CORRUPTION);
 495   if (db_status_ != SYNC_STATUS_OK) {
 496     db_.reset();
 497     return db_status_;
 498   }
 499
 500   leveldb::Status status = db_->Put(leveldb::WriteOptions(), url, kMark);
 501   if (!status.ok()) {
 502     HandleError(FROM_HERE, status);
 503     db_status_ = LevelDBStatusToSyncStatusCode(status);
 504     db_.reset();
 505     return db_status_;
 506   }
 507   return SYNC_STATUS_OK;
 508 }
 509
 510 SyncStatusCode LocalFileChangeTracker::TrackerDB::ClearDirty(
 511     const std::string& url) {
 512   if (db_status_ != SYNC_STATUS_OK)
 513     return db_status_;
 514
 515   // Should not reach here before initializing the database. The database should
 516   // be cleared after read, and should be initialized during read if
 517   // uninitialized.
 518   DCHECK(db_.get());
 519
 520   leveldb::Status status = db_->Delete(leveldb::WriteOptions(), url);
 521   if (!status.ok() && !status.IsNotFound()) {
 522     HandleError(FROM_HERE, status);
 523     db_status_ = LevelDBStatusToSyncStatusCode(status);
 524     db_.reset();
 525     return db_status_;
 526   }
 527   return SYNC_STATUS_OK;
 528 }
 529
 530 SyncStatusCode LocalFileChangeTracker::TrackerDB::GetDirtyEntries(
 531     std::queue<FileSystemURL>* dirty_files) {
 532   if (db_status_ != SYNC_STATUS_OK)
 533     return db_status_;
 534
 535   db_status_ = Init(REPAIR_ON_CORRUPTION);
 536   if (db_status_ != SYNC_STATUS_OK) {
 537     db_.reset();
 538     return db_status_;
 539   }
 540
 541   scoped_ptr<leveldb::Iterator> iter(db_->NewIterator(leveldb::ReadOptions()));
 542   iter->SeekToFirst();
 543   FileSystemURL url;
 544   while (iter->Valid()) {
 545     if (!DeserializeSyncableFileSystemURL(iter->key().ToString(), &url)) {
 546       LOG(WARNING) << "Failed to deserialize an URL. "
 547                    << "TrackerDB might be corrupted.";
 548       db_status_ = SYNC_DATABASE_ERROR_CORRUPTION;
 549       db_.reset();
 550       return db_status_;
 551     }
 552     dirty_files->push(url);
 553     iter->Next();
 554   }
 555   return SYNC_STATUS_OK;
 556 }
 557
 558 SyncStatusCode LocalFileChangeTracker::TrackerDB::WriteBatch(
 559     scoped_ptr<leveldb::WriteBatch> batch) {
 560   if (db_status_ != SYNC_STATUS_OK)
 561     return db_status_;
 562
 563   leveldb::Status status = db_->Write(leveldb::WriteOptions(), batch.get());
 564   if (!status.ok() && !status.IsNotFound()) {
 565     HandleError(FROM_HERE, status);
 566     db_status_ = LevelDBStatusToSyncStatusCode(status);
 567     db_.reset();
 568     return db_status_;
 569   }
 570   return SYNC_STATUS_OK;
 571 }
 572
 573 }  // namespace sync_file_system