sync/internal_api/attachments/on_disk_attachment_store.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "sync/internal_api/public/attachments/on_disk_attachment_store.h"
   6
   7 #include "base/bind.h"
   8 #include "base/callback.h"
   9 #include "base/location.h"
  10 #include "base/memory/scoped_ptr.h"
  11 #include "base/metrics/histogram.h"
  12 #include "base/sequenced_task_runner.h"
  13 #include "sync/internal_api/attachments/proto/attachment_store.pb.h"
  14 #include "sync/internal_api/public/attachments/attachment_util.h"
  15 #include "sync/protocol/attachments.pb.h"
  16 #include "third_party/leveldatabase/env_chromium.h"
  17 #include "third_party/leveldatabase/src/include/leveldb/db.h"
  18 #include "third_party/leveldatabase/src/include/leveldb/options.h"
  19 #include "third_party/leveldatabase/src/include/leveldb/slice.h"
  20 #include "third_party/leveldatabase/src/include/leveldb/status.h"
  21 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
  22
  23 namespace syncer {
  24
  25 namespace {
  26
  27 // Prefix for records containing attachment data.
  28 const char kDataPrefix[] = "data-";
  29
  30 // Prefix for records containing attachment metadata.
  31 const char kMetadataPrefix[] = "metadata-";
  32
  33 const char kDatabaseMetadataKey[] = "database-metadata";
  34
  35 const int32 kCurrentSchemaVersion = 1;
  36
  37 const base::FilePath::CharType kLeveldbDirectory[] =
  38     FILE_PATH_LITERAL("leveldb");
  39
  40 // Converts syncer::AttachmentStore::Component values into
  41 // attachment_store_pb::RecordMetadata::Component.
  42 attachment_store_pb::RecordMetadata::Component ComponentToProto(
  43     syncer::AttachmentStore::Component component) {
  44   switch (component) {
  45     case AttachmentStore::MODEL_TYPE:
  46       return attachment_store_pb::RecordMetadata::MODEL_TYPE;
  47     case AttachmentStore::SYNC:
  48       return attachment_store_pb::RecordMetadata::SYNC;
  49   }
  50   NOTREACHED();
  51   return attachment_store_pb::RecordMetadata::UNKNOWN;
  52 }
  53
  54 leveldb::WriteOptions MakeWriteOptions() {
  55   leveldb::WriteOptions write_options;
  56   write_options.sync = true;
  57   return write_options;
  58 }
  59
  60 leveldb::ReadOptions MakeNonCachingReadOptions() {
  61   leveldb::ReadOptions read_options;
  62   read_options.fill_cache = false;
  63   read_options.verify_checksums = true;
  64   return read_options;
  65 }
  66
  67 leveldb::ReadOptions MakeCachingReadOptions() {
  68   leveldb::ReadOptions read_options;
  69   read_options.fill_cache = true;
  70   read_options.verify_checksums = true;
  71   return read_options;
  72 }
  73
  74 leveldb::Status ReadStoreMetadata(
  75     leveldb::DB* db,
  76     attachment_store_pb::StoreMetadata* metadata) {
  77   std::string data_str;
  78
  79   leveldb::Status status =
  80       db->Get(MakeCachingReadOptions(), kDatabaseMetadataKey, &data_str);
  81   if (!status.ok())
  82     return status;
  83   if (!metadata->ParseFromString(data_str))
  84     return leveldb::Status::Corruption("Metadata record corruption");
  85   return leveldb::Status::OK();
  86 }
  87
  88 leveldb::Status WriteStoreMetadata(
  89     leveldb::DB* db,
  90     const attachment_store_pb::StoreMetadata& metadata) {
  91   std::string data_str;
  92
  93   metadata.SerializeToString(&data_str);
  94   return db->Put(MakeWriteOptions(), kDatabaseMetadataKey, data_str);
  95 }
  96
  97 // Adds reference to component into RecordMetadata::component set.
  98 // Returns true if record_metadata was modified and needs to be written to disk.
  99 bool SetReferenceInRecordMetadata(
 100     attachment_store_pb::RecordMetadata* record_metadata,
 101     attachment_store_pb::RecordMetadata::Component proto_component) {
 102   DCHECK(record_metadata);
 103   for (const int component : record_metadata->component()) {
 104     if (component == proto_component)
 105       return false;
 106   }
 107   record_metadata->add_component(proto_component);
 108   return true;
 109 }
 110
 111 // Drops reference to component from RecordMetadata::component set.
 112 // Returns true if record_metadata was modified and needs to be written to disk.
 113 bool DropReferenceInRecordMetadata(
 114     attachment_store_pb::RecordMetadata* record_metadata,
 115     attachment_store_pb::RecordMetadata::Component proto_component) {
 116   DCHECK(record_metadata);
 117   bool component_removed = false;
 118   ::google::protobuf::RepeatedField<int>* mutable_components =
 119       record_metadata->mutable_component();
 120   for (int i = 0; i < mutable_components->size();) {
 121     if (mutable_components->Get(i) == proto_component) {
 122       if (i < mutable_components->size() - 1) {
 123         // Don't swap last element with itself.
 124         mutable_components->SwapElements(i, mutable_components->size() - 1);
 125       }
 126       mutable_components->RemoveLast();
 127       component_removed = true;
 128     } else {
 129       ++i;
 130     }
 131   }
 132   return component_removed;
 133 }
 134
 135 }  // namespace
 136
 137 OnDiskAttachmentStore::OnDiskAttachmentStore(
 138     const scoped_refptr<base::SequencedTaskRunner>& callback_task_runner,
 139     const base::FilePath& path)
 140     : AttachmentStoreBackend(callback_task_runner), path_(path) {
 141 }
 142
 143 OnDiskAttachmentStore::~OnDiskAttachmentStore() {
 144 }
 145
 146 void OnDiskAttachmentStore::Init(
 147     const AttachmentStore::InitCallback& callback) {
 148   DCHECK(CalledOnValidThread());
 149   AttachmentStore::Result result_code = OpenOrCreate(path_);
 150   UMA_HISTOGRAM_ENUMERATION("Sync.Attachments.StoreInitResult", result_code,
 151                             AttachmentStore::RESULT_SIZE);
 152   PostCallback(base::Bind(callback, result_code));
 153 }
 154
 155 void OnDiskAttachmentStore::Read(
 156     const AttachmentIdList& ids,
 157     const AttachmentStore::ReadCallback& callback) {
 158   DCHECK(CalledOnValidThread());
 159   scoped_ptr<AttachmentMap> result_map(new AttachmentMap());
 160   scoped_ptr<AttachmentIdList> unavailable_attachments(new AttachmentIdList());
 161
 162   AttachmentStore::Result result_code =
 163       AttachmentStore::STORE_INITIALIZATION_FAILED;
 164
 165   if (db_) {
 166     result_code = AttachmentStore::SUCCESS;
 167     AttachmentIdList::const_iterator iter = ids.begin();
 168     const AttachmentIdList::const_iterator end = ids.end();
 169     for (; iter != end; ++iter) {
 170       scoped_ptr<Attachment> attachment;
 171       attachment = ReadSingleAttachment(*iter);
 172       if (attachment) {
 173         result_map->insert(std::make_pair(*iter, *attachment));
 174       } else {
 175         unavailable_attachments->push_back(*iter);
 176       }
 177     }
 178     result_code = unavailable_attachments->empty()
 179                       ? AttachmentStore::SUCCESS
 180                       : AttachmentStore::UNSPECIFIED_ERROR;
 181   } else {
 182     *unavailable_attachments = ids;
 183   }
 184
 185   PostCallback(base::Bind(callback, result_code, base::Passed(&result_map),
 186                           base::Passed(&unavailable_attachments)));
 187 }
 188
 189 void OnDiskAttachmentStore::Write(
 190     AttachmentStore::Component component,
 191     const AttachmentList& attachments,
 192     const AttachmentStore::WriteCallback& callback) {
 193   DCHECK(CalledOnValidThread());
 194   AttachmentStore::Result result_code =
 195       AttachmentStore::STORE_INITIALIZATION_FAILED;
 196
 197   if (db_) {
 198     result_code = AttachmentStore::SUCCESS;
 199     AttachmentList::const_iterator iter = attachments.begin();
 200     const AttachmentList::const_iterator end = attachments.end();
 201     for (; iter != end; ++iter) {
 202       if (!WriteSingleAttachment(*iter, component))
 203         result_code = AttachmentStore::UNSPECIFIED_ERROR;
 204     }
 205   }
 206   PostCallback(base::Bind(callback, result_code));
 207 }
 208
 209 void OnDiskAttachmentStore::SetReference(AttachmentStore::Component component,
 210                                          const AttachmentIdList& ids) {
 211   DCHECK(CalledOnValidThread());
 212   if (!db_)
 213     return;
 214   attachment_store_pb::RecordMetadata::Component proto_component =
 215       ComponentToProto(component);
 216   for (const auto& id : ids) {
 217     attachment_store_pb::RecordMetadata record_metadata;
 218     if (!ReadSingleRecordMetadata(id, &record_metadata))
 219       continue;
 220     if (SetReferenceInRecordMetadata(&record_metadata, proto_component))
 221       WriteSingleRecordMetadata(id, record_metadata);
 222   }
 223 }
 224
 225 void OnDiskAttachmentStore::DropReference(
 226     AttachmentStore::Component component,
 227     const AttachmentIdList& ids,
 228     const AttachmentStore::DropCallback& callback) {
 229   DCHECK(CalledOnValidThread());
 230   AttachmentStore::Result result_code =
 231       AttachmentStore::STORE_INITIALIZATION_FAILED;
 232   if (db_) {
 233     attachment_store_pb::RecordMetadata::Component proto_component =
 234         ComponentToProto(component);
 235     result_code = AttachmentStore::SUCCESS;
 236     leveldb::WriteOptions write_options = MakeWriteOptions();
 237     for (const auto& id : ids) {
 238       attachment_store_pb::RecordMetadata record_metadata;
 239       if (!ReadSingleRecordMetadata(id, &record_metadata))
 240         continue;  // Record not found.
 241       if (!DropReferenceInRecordMetadata(&record_metadata, proto_component))
 242         continue;  // Component is not in components set. Metadata was not
 243                    // updated.
 244       if (record_metadata.component_size() == 0) {
 245         // Last reference dropped. Need to delete attachment.
 246         leveldb::WriteBatch write_batch;
 247         write_batch.Delete(MakeDataKeyFromAttachmentId(id));
 248         write_batch.Delete(MakeMetadataKeyFromAttachmentId(id));
 249
 250         leveldb::Status status = db_->Write(write_options, &write_batch);
 251         if (!status.ok()) {
 252           // DB::Delete doesn't check if record exists, it returns ok just like
 253           // AttachmentStore::Drop should.
 254           DVLOG(1) << "DB::Write failed: status=" << status.ToString();
 255           result_code = AttachmentStore::UNSPECIFIED_ERROR;
 256         }
 257       } else {
 258         WriteSingleRecordMetadata(id, record_metadata);
 259       }
 260     }
 261   }
 262   PostCallback(base::Bind(callback, result_code));
 263 }
 264
 265 void OnDiskAttachmentStore::ReadMetadata(
 266     const AttachmentIdList& ids,
 267     const AttachmentStore::ReadMetadataCallback& callback) {
 268   DCHECK(CalledOnValidThread());
 269   AttachmentStore::Result result_code =
 270       AttachmentStore::STORE_INITIALIZATION_FAILED;
 271   scoped_ptr<AttachmentMetadataList> metadata_list(
 272       new AttachmentMetadataList());
 273   if (db_) {
 274     result_code = AttachmentStore::SUCCESS;
 275     // TODO(pavely): ReadMetadata should only return attachments with component
 276     // reference similarly to ReadAllMetadata behavior.
 277     for (const auto& id : ids) {
 278       attachment_store_pb::RecordMetadata record_metadata;
 279       if (ReadSingleRecordMetadata(id, &record_metadata)) {
 280         metadata_list->push_back(MakeAttachmentMetadata(id, record_metadata));
 281       } else {
 282         result_code = AttachmentStore::UNSPECIFIED_ERROR;
 283       }
 284     }
 285   }
 286   PostCallback(base::Bind(callback, result_code, base::Passed(&metadata_list)));
 287 }
 288
 289 void OnDiskAttachmentStore::ReadAllMetadata(
 290     AttachmentStore::Component component,
 291     const AttachmentStore::ReadMetadataCallback& callback) {
 292   DCHECK(CalledOnValidThread());
 293   AttachmentStore::Result result_code =
 294       AttachmentStore::STORE_INITIALIZATION_FAILED;
 295   scoped_ptr<AttachmentMetadataList> metadata_list(
 296       new AttachmentMetadataList());
 297
 298   if (db_) {
 299     attachment_store_pb::RecordMetadata::Component proto_component =
 300         ComponentToProto(component);
 301     result_code = AttachmentStore::SUCCESS;
 302     scoped_ptr<leveldb::Iterator> db_iterator(
 303         db_->NewIterator(MakeNonCachingReadOptions()));
 304     DCHECK(db_iterator);
 305     for (db_iterator->Seek(kMetadataPrefix); db_iterator->Valid();
 306          db_iterator->Next()) {
 307       leveldb::Slice key = db_iterator->key();
 308       if (!key.starts_with(kMetadataPrefix)) {
 309         break;
 310       }
 311       // Make AttachmentId from levelDB key.
 312       key.remove_prefix(strlen(kMetadataPrefix));
 313       sync_pb::AttachmentIdProto id_proto;
 314       id_proto.set_unique_id(key.ToString());
 315       AttachmentId id = AttachmentId::CreateFromProto(id_proto);
 316       // Parse metadata record.
 317       attachment_store_pb::RecordMetadata record_metadata;
 318       if (!record_metadata.ParseFromString(db_iterator->value().ToString())) {
 319         DVLOG(1) << "RecordMetadata::ParseFromString failed";
 320         result_code = AttachmentStore::UNSPECIFIED_ERROR;
 321         continue;
 322       }
 323       // Check if attachment has reference from component.
 324       bool has_reference_from_component = false;
 325       for (const auto& reference_component : record_metadata.component()) {
 326         if (reference_component == proto_component) {
 327           has_reference_from_component = true;
 328           break;
 329         }
 330       }
 331       if (has_reference_from_component)
 332         metadata_list->push_back(MakeAttachmentMetadata(id, record_metadata));
 333     }
 334
 335     if (!db_iterator->status().ok()) {
 336       DVLOG(1) << "DB Iterator failed: status="
 337                << db_iterator->status().ToString();
 338       result_code = AttachmentStore::UNSPECIFIED_ERROR;
 339     }
 340   }
 341
 342   PostCallback(base::Bind(callback, result_code, base::Passed(&metadata_list)));
 343 }
 344
 345 AttachmentStore::Result OnDiskAttachmentStore::OpenOrCreate(
 346     const base::FilePath& path) {
 347   DCHECK(!db_);
 348   base::FilePath leveldb_path = path.Append(kLeveldbDirectory);
 349
 350   leveldb::DB* db_raw;
 351   scoped_ptr<leveldb::DB> db;
 352   leveldb::Options options;
 353   options.create_if_missing = true;
 354   options.reuse_logs = leveldb_env::kDefaultLogReuseOptionValue;
 355   // TODO(pavely): crbug/424287 Consider adding info_log, block_cache and
 356   // filter_policy to options.
 357   leveldb::Status status =
 358       leveldb::DB::Open(options, leveldb_path.AsUTF8Unsafe(), &db_raw);
 359   if (!status.ok()) {
 360     DVLOG(1) << "DB::Open failed: status=" << status.ToString()
 361              << ", path=" << path.AsUTF8Unsafe();
 362     return AttachmentStore::UNSPECIFIED_ERROR;
 363   }
 364
 365   db.reset(db_raw);
 366
 367   attachment_store_pb::StoreMetadata metadata;
 368   status = ReadStoreMetadata(db.get(), &metadata);
 369   if (!status.ok() && !status.IsNotFound()) {
 370     DVLOG(1) << "ReadStoreMetadata failed: status=" << status.ToString();
 371     return AttachmentStore::UNSPECIFIED_ERROR;
 372   }
 373   if (status.IsNotFound()) {
 374     // Brand new database.
 375     metadata.set_schema_version(kCurrentSchemaVersion);
 376     status = WriteStoreMetadata(db.get(), metadata);
 377     if (!status.ok()) {
 378       DVLOG(1) << "WriteStoreMetadata failed: status=" << status.ToString();
 379       return AttachmentStore::UNSPECIFIED_ERROR;
 380     }
 381   }
 382   DCHECK(status.ok());
 383
 384   // Upgrade code goes here.
 385
 386   if (metadata.schema_version() != kCurrentSchemaVersion) {
 387     DVLOG(1) << "Unknown schema version: " << metadata.schema_version();
 388     return AttachmentStore::UNSPECIFIED_ERROR;
 389   }
 390
 391   db_ = db.Pass();
 392   return AttachmentStore::SUCCESS;
 393 }
 394
 395 scoped_ptr<Attachment> OnDiskAttachmentStore::ReadSingleAttachment(
 396     const AttachmentId& attachment_id) {
 397   scoped_ptr<Attachment> attachment;
 398   attachment_store_pb::RecordMetadata record_metadata;
 399   if (!ReadSingleRecordMetadata(attachment_id, &record_metadata)) {
 400     return attachment.Pass();
 401   }
 402   const std::string key = MakeDataKeyFromAttachmentId(attachment_id);
 403   std::string data_str;
 404   leveldb::Status status = db_->Get(
 405       MakeNonCachingReadOptions(), key, &data_str);
 406   if (!status.ok()) {
 407     DVLOG(1) << "DB::Get for data failed: status=" << status.ToString();
 408     return attachment.Pass();
 409   }
 410   scoped_refptr<base::RefCountedMemory> data =
 411       base::RefCountedString::TakeString(&data_str);
 412   uint32_t crc32c = ComputeCrc32c(data);
 413   if (record_metadata.has_crc32c()) {
 414     if (record_metadata.crc32c() != crc32c) {
 415       DVLOG(1) << "Attachment crc32c does not match value read from store";
 416       return attachment.Pass();
 417     }
 418     if (record_metadata.crc32c() != attachment_id.GetCrc32c()) {
 419       DVLOG(1) << "Attachment crc32c does not match value in AttachmentId";
 420       return attachment.Pass();
 421     }
 422   }
 423   attachment.reset(
 424       new Attachment(Attachment::CreateFromParts(attachment_id, data)));
 425   return attachment.Pass();
 426 }
 427
 428 bool OnDiskAttachmentStore::WriteSingleAttachment(
 429     const Attachment& attachment,
 430     AttachmentStore::Component component) {
 431   const std::string metadata_key =
 432       MakeMetadataKeyFromAttachmentId(attachment.GetId());
 433   const std::string data_key = MakeDataKeyFromAttachmentId(attachment.GetId());
 434
 435   std::string metadata_str;
 436   leveldb::Status status =
 437       db_->Get(MakeCachingReadOptions(), metadata_key, &metadata_str);
 438   if (status.ok()) {
 439     // Entry exists, don't overwrite.
 440     return true;
 441   } else if (!status.IsNotFound()) {
 442     // Entry exists but failed to read.
 443     DVLOG(1) << "DB::Get failed: status=" << status.ToString();
 444     return false;
 445   }
 446   DCHECK(status.IsNotFound());
 447
 448   leveldb::WriteBatch write_batch;
 449   // Write metadata.
 450   attachment_store_pb::RecordMetadata metadata;
 451   metadata.set_attachment_size(attachment.GetData()->size());
 452   metadata.set_crc32c(attachment.GetCrc32c());
 453   SetReferenceInRecordMetadata(&metadata, ComponentToProto(component));
 454   metadata_str = metadata.SerializeAsString();
 455   write_batch.Put(metadata_key, metadata_str);
 456   // Write data.
 457   scoped_refptr<base::RefCountedMemory> data = attachment.GetData();
 458   leveldb::Slice data_slice(data->front_as<char>(), data->size());
 459   write_batch.Put(data_key, data_slice);
 460
 461   status = db_->Write(MakeWriteOptions(), &write_batch);
 462   if (!status.ok()) {
 463     // Failed to write.
 464     DVLOG(1) << "DB::Write failed: status=" << status.ToString();
 465     return false;
 466   }
 467   return true;
 468 }
 469
 470 bool OnDiskAttachmentStore::ReadSingleRecordMetadata(
 471     const AttachmentId& attachment_id,
 472     attachment_store_pb::RecordMetadata* record_metadata) {
 473   DCHECK(record_metadata);
 474   const std::string metadata_key =
 475       MakeMetadataKeyFromAttachmentId(attachment_id);
 476   std::string metadata_str;
 477   leveldb::Status status =
 478       db_->Get(MakeCachingReadOptions(), metadata_key, &metadata_str);
 479   if (!status.ok()) {
 480     DVLOG(1) << "DB::Get for metadata failed: status=" << status.ToString();
 481     return false;
 482   }
 483   if (!record_metadata->ParseFromString(metadata_str)) {
 484     DVLOG(1) << "RecordMetadata::ParseFromString failed";
 485     return false;
 486   }
 487   return true;
 488 }
 489
 490 bool OnDiskAttachmentStore::WriteSingleRecordMetadata(
 491     const AttachmentId& attachment_id,
 492     const attachment_store_pb::RecordMetadata& record_metadata) {
 493   const std::string metadata_key =
 494       MakeMetadataKeyFromAttachmentId(attachment_id);
 495   std::string metadata_str;
 496   metadata_str = record_metadata.SerializeAsString();
 497   leveldb::Status status =
 498       db_->Put(MakeWriteOptions(), metadata_key, metadata_str);
 499   if (!status.ok()) {
 500     DVLOG(1) << "DB::Put failed: status=" << status.ToString();
 501     return false;
 502   }
 503   return true;
 504 }
 505
 506 std::string OnDiskAttachmentStore::MakeDataKeyFromAttachmentId(
 507     const AttachmentId& attachment_id) {
 508   std::string key = kDataPrefix + attachment_id.GetProto().unique_id();
 509   return key;
 510 }
 511
 512 std::string OnDiskAttachmentStore::MakeMetadataKeyFromAttachmentId(
 513     const AttachmentId& attachment_id) {
 514   std::string key = kMetadataPrefix + attachment_id.GetProto().unique_id();
 515   return key;
 516 }
 517
 518 AttachmentMetadata OnDiskAttachmentStore::MakeAttachmentMetadata(
 519     const AttachmentId& attachment_id,
 520     const attachment_store_pb::RecordMetadata& record_metadata) {
 521   return AttachmentMetadata(attachment_id, record_metadata.attachment_size());
 522 }
 523
 524 }  // namespace syncer