components/drive/change_list_processor.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/drive/change_list_processor.h"
   6
   7 #include "base/metrics/histogram.h"
   8 #include "base/strings/string_number_conversions.h"
   9 #include "base/synchronization/cancellation_flag.h"
  10 #include "components/drive/drive.pb.h"
  11 #include "components/drive/drive_api_util.h"
  12 #include "components/drive/file_change.h"
  13 #include "components/drive/file_system_core_util.h"
  14 #include "components/drive/resource_entry_conversion.h"
  15 #include "components/drive/resource_metadata.h"
  16 #include "google_apis/drive/drive_api_parser.h"
  17
  18 namespace drive {
  19 namespace internal {
  20
  21 namespace {
  22
  23 class ChangeListToEntryMapUMAStats {
  24  public:
  25   ChangeListToEntryMapUMAStats()
  26     : num_regular_files_(0),
  27       num_hosted_documents_(0),
  28       num_shared_with_me_entries_(0) {
  29   }
  30
  31   // Increments number of files.
  32   void IncrementNumFiles(bool is_hosted_document) {
  33     is_hosted_document ? num_hosted_documents_++ : num_regular_files_++;
  34   }
  35
  36   // Increments number of shared-with-me entries.
  37   void IncrementNumSharedWithMeEntries() {
  38     num_shared_with_me_entries_++;
  39   }
  40
  41   // Updates UMA histograms with file counts.
  42   void UpdateFileCountUmaHistograms() {
  43     const int num_total_files = num_hosted_documents_ + num_regular_files_;
  44     UMA_HISTOGRAM_COUNTS("Drive.NumberOfRegularFiles", num_regular_files_);
  45     UMA_HISTOGRAM_COUNTS("Drive.NumberOfHostedDocuments",
  46                          num_hosted_documents_);
  47     UMA_HISTOGRAM_COUNTS("Drive.NumberOfTotalFiles", num_total_files);
  48     UMA_HISTOGRAM_COUNTS("Drive.NumberOfSharedWithMeEntries",
  49                          num_shared_with_me_entries_);
  50   }
  51
  52  private:
  53   int num_regular_files_;
  54   int num_hosted_documents_;
  55   int num_shared_with_me_entries_;
  56 };
  57
  58 // Returns true if it's OK to overwrite the local entry with the remote one.
  59 bool ShouldApplyChange(const ResourceEntry& local_entry,
  60                        const ResourceEntry& remote_entry) {
  61   if (local_entry.metadata_edit_state() == ResourceEntry::CLEAN)
  62     return true;
  63   return base::Time::FromInternalValue(remote_entry.modification_date()) >
  64       base::Time::FromInternalValue(local_entry.modification_date());
  65 }
  66
  67 }  // namespace
  68
  69 std::string DirectoryFetchInfo::ToString() const {
  70   return ("local_id: " + local_id_ +
  71           ", resource_id: " + resource_id_ +
  72           ", changestamp: " + base::Int64ToString(changestamp_));
  73 }
  74
  75 ChangeList::ChangeList() {}
  76
  77 ChangeList::ChangeList(const google_apis::ChangeList& change_list)
  78     : next_url_(change_list.next_link()),
  79       largest_changestamp_(change_list.largest_change_id()) {
  80   const ScopedVector<google_apis::ChangeResource>& items = change_list.items();
  81   entries_.resize(items.size());
  82   parent_resource_ids_.resize(items.size());
  83   size_t entries_index = 0;
  84   for (size_t i = 0; i < items.size(); ++i) {
  85     if (ConvertChangeResourceToResourceEntry(
  86             *items[i],
  87             &entries_[entries_index],
  88             &parent_resource_ids_[entries_index])) {
  89       ++entries_index;
  90     }
  91   }
  92   entries_.resize(entries_index);
  93   parent_resource_ids_.resize(entries_index);
  94 }
  95
  96 ChangeList::ChangeList(const google_apis::FileList& file_list)
  97     : next_url_(file_list.next_link()),
  98       largest_changestamp_(0) {
  99   const ScopedVector<google_apis::FileResource>& items = file_list.items();
 100   entries_.resize(items.size());
 101   parent_resource_ids_.resize(items.size());
 102   size_t entries_index = 0;
 103   for (size_t i = 0; i < items.size(); ++i) {
 104     if (ConvertFileResourceToResourceEntry(
 105             *items[i],
 106             &entries_[entries_index],
 107             &parent_resource_ids_[entries_index])) {
 108       ++entries_index;
 109     }
 110   }
 111   entries_.resize(entries_index);
 112   parent_resource_ids_.resize(entries_index);
 113 }
 114
 115 ChangeList::~ChangeList() {}
 116
 117 ChangeListProcessor::ChangeListProcessor(ResourceMetadata* resource_metadata,
 118                                          base::CancellationFlag* in_shutdown)
 119     : resource_metadata_(resource_metadata),
 120       in_shutdown_(in_shutdown),
 121       changed_files_(new FileChange) {
 122 }
 123
 124 ChangeListProcessor::~ChangeListProcessor() {
 125 }
 126
 127 FileError ChangeListProcessor::Apply(
 128     scoped_ptr<google_apis::AboutResource> about_resource,
 129     ScopedVector<ChangeList> change_lists,
 130     bool is_delta_update) {
 131   DCHECK(about_resource);
 132
 133   int64 largest_changestamp = 0;
 134   if (is_delta_update) {
 135     if (!change_lists.empty()) {
 136       // The changestamp appears in the first page of the change list.
 137       // The changestamp does not appear in the full resource list.
 138       largest_changestamp = change_lists[0]->largest_changestamp();
 139       DCHECK_GE(change_lists[0]->largest_changestamp(), 0);
 140     }
 141   } else {
 142     largest_changestamp = about_resource->largest_change_id();
 143
 144     DVLOG(1) << "Root folder ID is " << about_resource->root_folder_id();
 145     DCHECK(!about_resource->root_folder_id().empty());
 146   }
 147
 148   // Convert ChangeList to map.
 149   ChangeListToEntryMapUMAStats uma_stats;
 150   for (size_t i = 0; i < change_lists.size(); ++i) {
 151     ChangeList* change_list = change_lists[i];
 152
 153     std::vector<ResourceEntry>* entries = change_list->mutable_entries();
 154     for (size_t i = 0; i < entries->size(); ++i) {
 155       ResourceEntry* entry = &(*entries)[i];
 156
 157       // Count the number of files.
 158       if (!entry->file_info().is_directory()) {
 159         uma_stats.IncrementNumFiles(
 160             entry->file_specific_info().is_hosted_document());
 161         if (entry->shared_with_me())
 162           uma_stats.IncrementNumSharedWithMeEntries();
 163       }
 164       parent_resource_id_map_[entry->resource_id()] =
 165           change_list->parent_resource_ids()[i];
 166       entry_map_[entry->resource_id()].Swap(entry);
 167       LOG_IF(WARNING, !entry->resource_id().empty())
 168           << "Found duplicated file: " << entry->base_name();
 169     }
 170   }
 171
 172   // Add the largest changestamp for directories.
 173   for (ResourceEntryMap::iterator it = entry_map_.begin();
 174        it != entry_map_.end(); ++it) {
 175     if (it->second.file_info().is_directory()) {
 176       it->second.mutable_directory_specific_info()->set_changestamp(
 177           largest_changestamp);
 178     }
 179   }
 180
 181   FileError error = ApplyEntryMap(largest_changestamp, about_resource.Pass());
 182   if (error != FILE_ERROR_OK) {
 183     DLOG(ERROR) << "ApplyEntryMap failed: " << FileErrorToString(error);
 184     return error;
 185   }
 186
 187   // Update changestamp.
 188   error = resource_metadata_->SetLargestChangestamp(largest_changestamp);
 189   if (error != FILE_ERROR_OK) {
 190     DLOG(ERROR) << "SetLargestChangeStamp failed: " << FileErrorToString(error);
 191     return error;
 192   }
 193
 194   // Shouldn't record histograms when processing delta update.
 195   if (!is_delta_update)
 196     uma_stats.UpdateFileCountUmaHistograms();
 197
 198   return FILE_ERROR_OK;
 199 }
 200
 201 FileError ChangeListProcessor::ApplyEntryMap(
 202     int64 changestamp,
 203     scoped_ptr<google_apis::AboutResource> about_resource) {
 204   DCHECK(about_resource);
 205
 206   // Create the entry for "My Drive" directory with the latest changestamp.
 207   ResourceEntry root;
 208   FileError error = resource_metadata_->GetResourceEntryByPath(
 209       util::GetDriveMyDriveRootPath(), &root);
 210   if (error != FILE_ERROR_OK) {
 211     LOG(ERROR) << "Failed to get root entry: " << FileErrorToString(error);
 212     return error;
 213   }
 214
 215   root.mutable_directory_specific_info()->set_changestamp(changestamp);
 216   root.set_resource_id(about_resource->root_folder_id());
 217   error = resource_metadata_->RefreshEntry(root);
 218   if (error != FILE_ERROR_OK) {
 219     LOG(ERROR) << "Failed to update root entry: " << FileErrorToString(error);
 220     return error;
 221   }
 222
 223   // Gather the set of changes in the old path.
 224   // Note that we want to notify the change in both old and new paths (suppose
 225   // /a/b/c is moved to /x/y/c. We want to notify both "/a/b" and "/x/y".)
 226   // The old paths must be calculated before we apply any actual changes.
 227   // The new paths are calculated after each change is applied. It correctly
 228   // sets the new path because we apply changes in such an order (see below).
 229   for (ResourceEntryMap::iterator it = entry_map_.begin();
 230        it != entry_map_.end(); ++it) {
 231     UpdateChangedDirs(it->second);
 232   }
 233
 234   // Apply all entries except deleted ones to the metadata.
 235   std::vector<std::string> deleted_resource_ids;
 236   while (!entry_map_.empty()) {
 237     if (in_shutdown_ && in_shutdown_->IsSet())
 238       return FILE_ERROR_ABORT;
 239
 240     ResourceEntryMap::iterator it = entry_map_.begin();
 241
 242     // Process deleted entries later to avoid deleting moved entries under it.
 243     if (it->second.deleted()) {
 244       deleted_resource_ids.push_back(it->first);
 245       entry_map_.erase(it);
 246       continue;
 247     }
 248
 249     // Start from entry_map_.begin() and traverse ancestors using the
 250     // parent-child relationships in the result (after this apply) tree.
 251     // Then apply the topmost change first.
 252     //
 253     // By doing this, assuming the result tree does not contain any cycles, we
 254     // can guarantee that no cycle is made during this apply (i.e. no entry gets
 255     // moved under any of its descendants) because the following conditions are
 256     // always satisfied in any move:
 257     // - The new parent entry is not a descendant of the moved entry.
 258     // - The new parent and its ancestors will no longer move during this apply.
 259     std::vector<ResourceEntryMap::iterator> entries;
 260     for (ResourceEntryMap::iterator it = entry_map_.begin();
 261          it != entry_map_.end();) {
 262       entries.push_back(it);
 263
 264       DCHECK(parent_resource_id_map_.count(it->first)) << it->first;
 265       const std::string& parent_resource_id =
 266           parent_resource_id_map_[it->first];
 267
 268       if (parent_resource_id.empty())  // This entry has no parent.
 269         break;
 270
 271       ResourceEntryMap::iterator it_parent =
 272           entry_map_.find(parent_resource_id);
 273       if (it_parent == entry_map_.end()) {
 274         // Current entry's parent is already updated or not going to be updated,
 275         // get the parent from the local tree.
 276         std::string parent_local_id;
 277         FileError error = resource_metadata_->GetIdByResourceId(
 278             parent_resource_id, &parent_local_id);
 279         if (error != FILE_ERROR_OK) {
 280           // See crbug.com/326043. In some complicated situations, parent folder
 281           // for shared entries may be accessible (and hence its resource id is
 282           // included), but not in the change/file list.
 283           // In such a case, clear the parent and move it to drive/other.
 284           if (error == FILE_ERROR_NOT_FOUND) {
 285             parent_resource_id_map_[it->first] = "";
 286           } else {
 287             LOG(ERROR) << "Failed to get local ID: " << parent_resource_id
 288                        << ", error = " << FileErrorToString(error);
 289           }
 290           break;
 291         }
 292         ResourceEntry parent_entry;
 293         while (it_parent == entry_map_.end() && !parent_local_id.empty()) {
 294           error = resource_metadata_->GetResourceEntryById(
 295               parent_local_id, &parent_entry);
 296           if (error != FILE_ERROR_OK) {
 297             LOG(ERROR) << "Failed to get local entry: "
 298                        << FileErrorToString(error);
 299             break;
 300           }
 301           it_parent = entry_map_.find(parent_entry.resource_id());
 302           parent_local_id = parent_entry.parent_local_id();
 303         }
 304       }
 305       it = it_parent;
 306     }
 307
 308     // Apply the parent first.
 309     std::reverse(entries.begin(), entries.end());
 310     for (size_t i = 0; i < entries.size(); ++i) {
 311       // Skip root entry in the change list. We don't expect servers to send
 312       // root entry, but we should better be defensive (see crbug.com/297259).
 313       ResourceEntryMap::iterator it = entries[i];
 314       if (it->first != root.resource_id()) {
 315         FileError error = ApplyEntry(it->second);
 316         if (error != FILE_ERROR_OK) {
 317           LOG(ERROR) << "ApplyEntry failed: " << FileErrorToString(error)
 318                      << ", title = " << it->second.title();
 319           return error;
 320         }
 321       }
 322       entry_map_.erase(it);
 323     }
 324   }
 325
 326   // Apply deleted entries.
 327   for (size_t i = 0; i < deleted_resource_ids.size(); ++i) {
 328     std::string local_id;
 329     FileError error = resource_metadata_->GetIdByResourceId(
 330         deleted_resource_ids[i], &local_id);
 331     switch (error) {
 332       case FILE_ERROR_OK:
 333         error = resource_metadata_->RemoveEntry(local_id);
 334         break;
 335       case FILE_ERROR_NOT_FOUND:
 336         error = FILE_ERROR_OK;
 337         break;
 338       default:
 339         break;
 340     }
 341     if (error != FILE_ERROR_OK) {
 342       LOG(ERROR) << "Failed to delete: " << FileErrorToString(error)
 343                  << ", resource_id = " << deleted_resource_ids[i];
 344       return error;
 345     }
 346   }
 347
 348   return FILE_ERROR_OK;
 349 }
 350
 351 FileError ChangeListProcessor::ApplyEntry(const ResourceEntry& entry) {
 352   DCHECK(!entry.deleted());
 353   DCHECK(parent_resource_id_map_.count(entry.resource_id()));
 354   const std::string& parent_resource_id =
 355       parent_resource_id_map_[entry.resource_id()];
 356
 357   ResourceEntry new_entry(entry);
 358   FileError error = SetParentLocalIdOfEntry(resource_metadata_, &new_entry,
 359                                             parent_resource_id);
 360   if (error != FILE_ERROR_OK)
 361     return error;
 362
 363   // Lookup the entry.
 364   std::string local_id;
 365   error = resource_metadata_->GetIdByResourceId(entry.resource_id(), &local_id);
 366
 367   ResourceEntry existing_entry;
 368   if (error == FILE_ERROR_OK)
 369     error = resource_metadata_->GetResourceEntryById(local_id, &existing_entry);
 370
 371   switch (error) {
 372     case FILE_ERROR_OK:
 373       if (ShouldApplyChange(existing_entry, new_entry)) {
 374         // Entry exists and needs to be refreshed.
 375         new_entry.set_local_id(local_id);
 376         // Keep the to-be-synced properties of the existing resource entry.
 377         new_entry.mutable_new_properties()->CopyFrom(
 378             existing_entry.new_properties());
 379         error = resource_metadata_->RefreshEntry(new_entry);
 380       } else {
 381         if (entry.file_info().is_directory()) {
 382           // No need to refresh, but update the changestamp.
 383           new_entry = existing_entry;
 384           new_entry.mutable_directory_specific_info()->set_changestamp(
 385               new_entry.directory_specific_info().changestamp());
 386           error = resource_metadata_->RefreshEntry(new_entry);
 387         }
 388         DVLOG(1) << "Change was discarded for: " << entry.resource_id();
 389       }
 390       break;
 391     case FILE_ERROR_NOT_FOUND: {  // Adding a new entry.
 392       std::string local_id;
 393       error = resource_metadata_->AddEntry(new_entry, &local_id);
 394       break;
 395     }
 396     default:
 397       return error;
 398   }
 399   if (error != FILE_ERROR_OK)
 400     return error;
 401
 402   UpdateChangedDirs(entry);
 403   return FILE_ERROR_OK;
 404 }
 405
 406 // static
 407 FileError ChangeListProcessor::RefreshDirectory(
 408     ResourceMetadata* resource_metadata,
 409     const DirectoryFetchInfo& directory_fetch_info,
 410     scoped_ptr<ChangeList> change_list,
 411     std::vector<ResourceEntry>* out_refreshed_entries) {
 412   DCHECK(!directory_fetch_info.empty());
 413
 414   ResourceEntry directory;
 415   FileError error = resource_metadata->GetResourceEntryById(
 416       directory_fetch_info.local_id(), &directory);
 417   if (error != FILE_ERROR_OK)
 418     return error;
 419
 420   if (!directory.file_info().is_directory())
 421     return FILE_ERROR_NOT_A_DIRECTORY;
 422
 423   std::vector<ResourceEntry>* entries = change_list->mutable_entries();
 424   for (size_t i = 0; i < entries->size(); ++i) {
 425     ResourceEntry* entry = &(*entries)[i];
 426     const std::string& parent_resource_id =
 427         change_list->parent_resource_ids()[i];
 428
 429     // Skip if the parent resource ID does not match. This is needed to
 430     // handle entries with multiple parents. For such entries, the first
 431     // parent is picked and other parents are ignored, hence some entries may
 432     // have a parent resource ID which does not match the target directory's.
 433     if (parent_resource_id != directory_fetch_info.resource_id()) {
 434       DVLOG(1) << "Wrong-parent entry rejected: " << entry->resource_id();
 435       continue;
 436     }
 437
 438     entry->set_parent_local_id(directory_fetch_info.local_id());
 439
 440     std::string local_id;
 441     error = resource_metadata->GetIdByResourceId(entry->resource_id(),
 442                                                  &local_id);
 443     if (error == FILE_ERROR_OK) {
 444       entry->set_local_id(local_id);
 445       error = resource_metadata->RefreshEntry(*entry);
 446     }
 447
 448     if (error == FILE_ERROR_NOT_FOUND) {  // If refreshing fails, try adding.
 449       entry->clear_local_id();
 450       error = resource_metadata->AddEntry(*entry, &local_id);
 451     }
 452
 453     if (error != FILE_ERROR_OK)
 454       return error;
 455
 456     ResourceEntry result_entry;
 457     error = resource_metadata->GetResourceEntryById(local_id, &result_entry);
 458     if (error != FILE_ERROR_OK)
 459       return error;
 460     out_refreshed_entries->push_back(result_entry);
 461   }
 462   return FILE_ERROR_OK;
 463 }
 464
 465 // static
 466 FileError ChangeListProcessor::SetParentLocalIdOfEntry(
 467     ResourceMetadata* resource_metadata,
 468     ResourceEntry* entry,
 469     const std::string& parent_resource_id) {
 470   std::string parent_local_id;
 471   if (parent_resource_id.empty()) {
 472     // Entries without parents should go under "other" directory.
 473     parent_local_id = util::kDriveOtherDirLocalId;
 474   } else {
 475     FileError error = resource_metadata->GetIdByResourceId(
 476         parent_resource_id, &parent_local_id);
 477     if (error != FILE_ERROR_OK)
 478       return error;
 479   }
 480   entry->set_parent_local_id(parent_local_id);
 481   return FILE_ERROR_OK;
 482 }
 483
 484 void ChangeListProcessor::UpdateChangedDirs(const ResourceEntry& entry) {
 485   DCHECK(!entry.resource_id().empty());
 486
 487   std::string local_id;
 488   base::FilePath file_path;
 489   if (resource_metadata_->GetIdByResourceId(
 490           entry.resource_id(), &local_id) == FILE_ERROR_OK)
 491     resource_metadata_->GetFilePath(local_id, &file_path);
 492
 493   if (!file_path.empty()) {
 494     FileChange::ChangeType type = entry.deleted()
 495                                       ? FileChange::CHANGE_TYPE_DELETE
 496                                       : FileChange::CHANGE_TYPE_ADD_OR_UPDATE;
 497     changed_files_->Update(file_path, entry, type);
 498   }
 499 }
 500
 501 }  // namespace internal
 502 }  // namespace drive