sync/internal_api/attachments/on_disk_attachment_store.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "sync/internal_api/public/attachments/on_disk_attachment_store.h"
   6
   7 #include "base/bind.h"
   8 #include "base/callback.h"
   9 #include "base/location.h"
  10 #include "base/memory/scoped_ptr.h"
  11 #include "base/metrics/histogram.h"
  12 #include "base/sequenced_task_runner.h"
  13 #include "sync/internal_api/attachments/proto/attachment_store.pb.h"
  14 #include "sync/internal_api/public/attachments/attachment_util.h"
  15 #include "sync/protocol/attachments.pb.h"
  16 #include "third_party/leveldatabase/env_chromium.h"
  17 #include "third_party/leveldatabase/src/include/leveldb/db.h"
  18 #include "third_party/leveldatabase/src/include/leveldb/options.h"
  19 #include "third_party/leveldatabase/src/include/leveldb/slice.h"
  20 #include "third_party/leveldatabase/src/include/leveldb/status.h"
  21 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
  22
  23 namespace syncer {
  24
  25 namespace {
  26
  27 // Prefix for records containing attachment data.
  28 const char kDataPrefix[] = "data-";
  29
  30 // Prefix for records containing attachment metadata.
  31 const char kMetadataPrefix[] = "metadata-";
  32
  33 const char kDatabaseMetadataKey[] = "database-metadata";
  34
  35 const int32 kCurrentSchemaVersion = 1;
  36
  37 const base::FilePath::CharType kLeveldbDirectory[] =
  38     FILE_PATH_LITERAL("leveldb");
  39
  40 leveldb::WriteOptions MakeWriteOptions() {
  41   leveldb::WriteOptions write_options;
  42   write_options.sync = true;
  43   return write_options;
  44 }
  45
  46 leveldb::ReadOptions MakeNonCachingReadOptions() {
  47   leveldb::ReadOptions read_options;
  48   read_options.fill_cache = false;
  49   read_options.verify_checksums = true;
  50   return read_options;
  51 }
  52
  53 leveldb::ReadOptions MakeCachingReadOptions() {
  54   leveldb::ReadOptions read_options;
  55   read_options.fill_cache = true;
  56   read_options.verify_checksums = true;
  57   return read_options;
  58 }
  59
  60 leveldb::Status ReadStoreMetadata(
  61     leveldb::DB* db,
  62     attachment_store_pb::StoreMetadata* metadata) {
  63   std::string data_str;
  64
  65   leveldb::Status status =
  66       db->Get(MakeCachingReadOptions(), kDatabaseMetadataKey, &data_str);
  67   if (!status.ok())
  68     return status;
  69   if (!metadata->ParseFromString(data_str))
  70     return leveldb::Status::Corruption("Metadata record corruption");
  71   return leveldb::Status::OK();
  72 }
  73
  74 leveldb::Status WriteStoreMetadata(
  75     leveldb::DB* db,
  76     const attachment_store_pb::StoreMetadata& metadata) {
  77   std::string data_str;
  78
  79   metadata.SerializeToString(&data_str);
  80   return db->Put(MakeWriteOptions(), kDatabaseMetadataKey, data_str);
  81 }
  82
  83 }  // namespace
  84
  85 OnDiskAttachmentStore::OnDiskAttachmentStore(
  86     const scoped_refptr<base::SequencedTaskRunner>& callback_task_runner,
  87     const base::FilePath& path)
  88     : AttachmentStoreBackend(callback_task_runner), path_(path) {
  89 }
  90
  91 OnDiskAttachmentStore::~OnDiskAttachmentStore() {
  92 }
  93
  94 void OnDiskAttachmentStore::Init(
  95     const AttachmentStore::InitCallback& callback) {
  96   DCHECK(CalledOnValidThread());
  97   AttachmentStore::Result result_code = OpenOrCreate(path_);
  98   UMA_HISTOGRAM_ENUMERATION("Sync.Attachments.StoreInitResult", result_code,
  99                             AttachmentStore::RESULT_SIZE);
 100   PostCallback(base::Bind(callback, result_code));
 101 }
 102
 103 void OnDiskAttachmentStore::Read(
 104     const AttachmentIdList& ids,
 105     const AttachmentStore::ReadCallback& callback) {
 106   DCHECK(CalledOnValidThread());
 107   scoped_ptr<AttachmentMap> result_map(new AttachmentMap());
 108   scoped_ptr<AttachmentIdList> unavailable_attachments(new AttachmentIdList());
 109
 110   AttachmentStore::Result result_code =
 111       AttachmentStore::STORE_INITIALIZATION_FAILED;
 112
 113   if (db_) {
 114     result_code = AttachmentStore::SUCCESS;
 115     AttachmentIdList::const_iterator iter = ids.begin();
 116     const AttachmentIdList::const_iterator end = ids.end();
 117     for (; iter != end; ++iter) {
 118       scoped_ptr<Attachment> attachment;
 119       attachment = ReadSingleAttachment(*iter);
 120       if (attachment) {
 121         result_map->insert(std::make_pair(*iter, *attachment));
 122       } else {
 123         unavailable_attachments->push_back(*iter);
 124       }
 125     }
 126     result_code = unavailable_attachments->empty()
 127                       ? AttachmentStore::SUCCESS
 128                       : AttachmentStore::UNSPECIFIED_ERROR;
 129   } else {
 130     *unavailable_attachments = ids;
 131   }
 132
 133   PostCallback(base::Bind(callback, result_code, base::Passed(&result_map),
 134                           base::Passed(&unavailable_attachments)));
 135 }
 136
 137 void OnDiskAttachmentStore::Write(
 138     AttachmentStore::Component component,
 139     const AttachmentList& attachments,
 140     const AttachmentStore::WriteCallback& callback) {
 141   DCHECK(CalledOnValidThread());
 142   AttachmentStore::Result result_code =
 143       AttachmentStore::STORE_INITIALIZATION_FAILED;
 144
 145   if (db_) {
 146     result_code = AttachmentStore::SUCCESS;
 147     AttachmentList::const_iterator iter = attachments.begin();
 148     const AttachmentList::const_iterator end = attachments.end();
 149     for (; iter != end; ++iter) {
 150       if (!WriteSingleAttachment(*iter))
 151         result_code = AttachmentStore::UNSPECIFIED_ERROR;
 152     }
 153   }
 154   PostCallback(base::Bind(callback, result_code));
 155 }
 156
 157 void OnDiskAttachmentStore::SetReference(AttachmentStore::Component component,
 158                                          const AttachmentIdList& ids) {
 159   DCHECK(CalledOnValidThread());
 160   DCHECK_EQ(AttachmentStore::SYNC, component);
 161 }
 162
 163 void OnDiskAttachmentStore::DropReference(
 164     AttachmentStore::Component component,
 165     const AttachmentIdList& ids,
 166     const AttachmentStore::DropCallback& callback) {
 167   DCHECK(CalledOnValidThread());
 168   if (component == AttachmentStore::SYNC) {
 169     // TODO(pavely): There is no reference handling implementation yet. All
 170     // calls to AddReferrer are ignored. Calls to Drop coming from sync should
 171     // be ignored too.
 172     PostCallback(base::Bind(callback, AttachmentStore::SUCCESS));
 173     return;
 174   }
 175   AttachmentStore::Result result_code =
 176       AttachmentStore::STORE_INITIALIZATION_FAILED;
 177   if (db_) {
 178     result_code = AttachmentStore::SUCCESS;
 179     leveldb::WriteOptions write_options = MakeWriteOptions();
 180     AttachmentIdList::const_iterator iter = ids.begin();
 181     const AttachmentIdList::const_iterator end = ids.end();
 182     for (; iter != end; ++iter) {
 183       leveldb::WriteBatch write_batch;
 184       write_batch.Delete(MakeDataKeyFromAttachmentId(*iter));
 185       write_batch.Delete(MakeMetadataKeyFromAttachmentId(*iter));
 186
 187       leveldb::Status status = db_->Write(write_options, &write_batch);
 188       if (!status.ok()) {
 189         // DB::Delete doesn't check if record exists, it returns ok just like
 190         // AttachmentStore::Drop should.
 191         DVLOG(1) << "DB::Write failed: status=" << status.ToString();
 192         result_code = AttachmentStore::UNSPECIFIED_ERROR;
 193       }
 194     }
 195   }
 196   PostCallback(base::Bind(callback, result_code));
 197 }
 198
 199 void OnDiskAttachmentStore::ReadMetadata(
 200     const AttachmentIdList& ids,
 201     const AttachmentStore::ReadMetadataCallback& callback) {
 202   DCHECK(CalledOnValidThread());
 203   AttachmentStore::Result result_code =
 204       AttachmentStore::STORE_INITIALIZATION_FAILED;
 205   scoped_ptr<AttachmentMetadataList> metadata_list(
 206       new AttachmentMetadataList());
 207   if (db_) {
 208     result_code = AttachmentStore::SUCCESS;
 209     AttachmentIdList::const_iterator iter = ids.begin();
 210     const AttachmentIdList::const_iterator end = ids.end();
 211     for (; iter != end; ++iter) {
 212       attachment_store_pb::RecordMetadata record_metadata;
 213       if (ReadSingleRecordMetadata(*iter, &record_metadata)) {
 214         metadata_list->push_back(
 215             MakeAttachmentMetadata(*iter, record_metadata));
 216       } else {
 217         result_code = AttachmentStore::UNSPECIFIED_ERROR;
 218       }
 219     }
 220   }
 221   PostCallback(base::Bind(callback, result_code, base::Passed(&metadata_list)));
 222 }
 223
 224 void OnDiskAttachmentStore::ReadAllMetadata(
 225     AttachmentStore::Component component,
 226     const AttachmentStore::ReadMetadataCallback& callback) {
 227   DCHECK(CalledOnValidThread());
 228   AttachmentStore::Result result_code =
 229       AttachmentStore::STORE_INITIALIZATION_FAILED;
 230   scoped_ptr<AttachmentMetadataList> metadata_list(
 231       new AttachmentMetadataList());
 232
 233   if (db_) {
 234     result_code = AttachmentStore::SUCCESS;
 235     scoped_ptr<leveldb::Iterator> db_iterator(
 236         db_->NewIterator(MakeNonCachingReadOptions()));
 237     DCHECK(db_iterator);
 238     for (db_iterator->Seek(kMetadataPrefix); db_iterator->Valid();
 239          db_iterator->Next()) {
 240       leveldb::Slice key = db_iterator->key();
 241       if (!key.starts_with(kMetadataPrefix)) {
 242         break;
 243       }
 244       // Make AttachmentId from levelDB key.
 245       key.remove_prefix(strlen(kMetadataPrefix));
 246       sync_pb::AttachmentIdProto id_proto;
 247       id_proto.set_unique_id(key.ToString());
 248       AttachmentId id = AttachmentId::CreateFromProto(id_proto);
 249       // Parse metadata record.
 250       attachment_store_pb::RecordMetadata record_metadata;
 251       if (!record_metadata.ParseFromString(db_iterator->value().ToString())) {
 252         DVLOG(1) << "RecordMetadata::ParseFromString failed";
 253         result_code = AttachmentStore::UNSPECIFIED_ERROR;
 254         continue;
 255       }
 256       metadata_list->push_back(MakeAttachmentMetadata(id, record_metadata));
 257     }
 258
 259     if (!db_iterator->status().ok()) {
 260       DVLOG(1) << "DB Iterator failed: status="
 261                << db_iterator->status().ToString();
 262       result_code = AttachmentStore::UNSPECIFIED_ERROR;
 263     }
 264   }
 265
 266   PostCallback(base::Bind(callback, result_code, base::Passed(&metadata_list)));
 267 }
 268
 269 AttachmentStore::Result OnDiskAttachmentStore::OpenOrCreate(
 270     const base::FilePath& path) {
 271   DCHECK(!db_);
 272   base::FilePath leveldb_path = path.Append(kLeveldbDirectory);
 273
 274   leveldb::DB* db_raw;
 275   scoped_ptr<leveldb::DB> db;
 276   leveldb::Options options;
 277   options.create_if_missing = true;
 278   options.reuse_logs = leveldb_env::kDefaultLogReuseOptionValue;
 279   // TODO(pavely): crbug/424287 Consider adding info_log, block_cache and
 280   // filter_policy to options.
 281   leveldb::Status status =
 282       leveldb::DB::Open(options, leveldb_path.AsUTF8Unsafe(), &db_raw);
 283   if (!status.ok()) {
 284     DVLOG(1) << "DB::Open failed: status=" << status.ToString()
 285              << ", path=" << path.AsUTF8Unsafe();
 286     return AttachmentStore::UNSPECIFIED_ERROR;
 287   }
 288
 289   db.reset(db_raw);
 290
 291   attachment_store_pb::StoreMetadata metadata;
 292   status = ReadStoreMetadata(db.get(), &metadata);
 293   if (!status.ok() && !status.IsNotFound()) {
 294     DVLOG(1) << "ReadStoreMetadata failed: status=" << status.ToString();
 295     return AttachmentStore::UNSPECIFIED_ERROR;
 296   }
 297   if (status.IsNotFound()) {
 298     // Brand new database.
 299     metadata.set_schema_version(kCurrentSchemaVersion);
 300     status = WriteStoreMetadata(db.get(), metadata);
 301     if (!status.ok()) {
 302       DVLOG(1) << "WriteStoreMetadata failed: status=" << status.ToString();
 303       return AttachmentStore::UNSPECIFIED_ERROR;
 304     }
 305   }
 306   DCHECK(status.ok());
 307
 308   // Upgrade code goes here.
 309
 310   if (metadata.schema_version() != kCurrentSchemaVersion) {
 311     DVLOG(1) << "Unknown schema version: " << metadata.schema_version();
 312     return AttachmentStore::UNSPECIFIED_ERROR;
 313   }
 314
 315   db_ = db.Pass();
 316   return AttachmentStore::SUCCESS;
 317 }
 318
 319 scoped_ptr<Attachment> OnDiskAttachmentStore::ReadSingleAttachment(
 320     const AttachmentId& attachment_id) {
 321   scoped_ptr<Attachment> attachment;
 322   attachment_store_pb::RecordMetadata record_metadata;
 323   if (!ReadSingleRecordMetadata(attachment_id, &record_metadata)) {
 324     return attachment.Pass();
 325   }
 326   const std::string key = MakeDataKeyFromAttachmentId(attachment_id);
 327   std::string data_str;
 328   leveldb::Status status = db_->Get(
 329       MakeNonCachingReadOptions(), key, &data_str);
 330   if (!status.ok()) {
 331     DVLOG(1) << "DB::Get for data failed: status=" << status.ToString();
 332     return attachment.Pass();
 333   }
 334   scoped_refptr<base::RefCountedMemory> data =
 335       base::RefCountedString::TakeString(&data_str);
 336   uint32_t crc32c = ComputeCrc32c(data);
 337   if (record_metadata.has_crc32c()) {
 338     if (record_metadata.crc32c() != crc32c) {
 339       DVLOG(1) << "Attachment crc32c does not match value read from store";
 340       return attachment.Pass();
 341     }
 342     if (record_metadata.crc32c() != attachment_id.GetCrc32c()) {
 343       DVLOG(1) << "Attachment crc32c does not match value in AttachmentId";
 344       return attachment.Pass();
 345     }
 346   }
 347   attachment.reset(
 348       new Attachment(Attachment::CreateFromParts(attachment_id, data)));
 349   return attachment.Pass();
 350 }
 351
 352 bool OnDiskAttachmentStore::WriteSingleAttachment(
 353     const Attachment& attachment) {
 354   const std::string metadata_key =
 355       MakeMetadataKeyFromAttachmentId(attachment.GetId());
 356   const std::string data_key = MakeDataKeyFromAttachmentId(attachment.GetId());
 357
 358   std::string metadata_str;
 359   leveldb::Status status =
 360       db_->Get(MakeCachingReadOptions(), metadata_key, &metadata_str);
 361   if (status.ok()) {
 362     // Entry exists, don't overwrite.
 363     return true;
 364   } else if (!status.IsNotFound()) {
 365     // Entry exists but failed to read.
 366     DVLOG(1) << "DB::Get failed: status=" << status.ToString();
 367     return false;
 368   }
 369   DCHECK(status.IsNotFound());
 370
 371   leveldb::WriteBatch write_batch;
 372   // Write metadata.
 373   attachment_store_pb::RecordMetadata metadata;
 374   metadata.set_attachment_size(attachment.GetData()->size());
 375   metadata.set_crc32c(attachment.GetCrc32c());
 376   metadata_str = metadata.SerializeAsString();
 377   write_batch.Put(metadata_key, metadata_str);
 378   // Write data.
 379   scoped_refptr<base::RefCountedMemory> data = attachment.GetData();
 380   leveldb::Slice data_slice(data->front_as<char>(), data->size());
 381   write_batch.Put(data_key, data_slice);
 382
 383   status = db_->Write(MakeWriteOptions(), &write_batch);
 384   if (!status.ok()) {
 385     // Failed to write.
 386     DVLOG(1) << "DB::Write failed: status=" << status.ToString();
 387     return false;
 388   }
 389   return true;
 390 }
 391
 392 bool OnDiskAttachmentStore::ReadSingleRecordMetadata(
 393     const AttachmentId& attachment_id,
 394     attachment_store_pb::RecordMetadata* record_metadata) {
 395   DCHECK(record_metadata);
 396   const std::string metadata_key =
 397       MakeMetadataKeyFromAttachmentId(attachment_id);
 398   std::string metadata_str;
 399   leveldb::Status status =
 400       db_->Get(MakeCachingReadOptions(), metadata_key, &metadata_str);
 401   if (!status.ok()) {
 402     DVLOG(1) << "DB::Get for metadata failed: status=" << status.ToString();
 403     return false;
 404   }
 405   if (!record_metadata->ParseFromString(metadata_str)) {
 406     DVLOG(1) << "RecordMetadata::ParseFromString failed";
 407     return false;
 408   }
 409   return true;
 410 }
 411
 412 std::string OnDiskAttachmentStore::MakeDataKeyFromAttachmentId(
 413     const AttachmentId& attachment_id) {
 414   std::string key = kDataPrefix + attachment_id.GetProto().unique_id();
 415   return key;
 416 }
 417
 418 std::string OnDiskAttachmentStore::MakeMetadataKeyFromAttachmentId(
 419     const AttachmentId& attachment_id) {
 420   std::string key = kMetadataPrefix + attachment_id.GetProto().unique_id();
 421   return key;
 422 }
 423
 424 AttachmentMetadata OnDiskAttachmentStore::MakeAttachmentMetadata(
 425     const AttachmentId& attachment_id,
 426     const attachment_store_pb::RecordMetadata& record_metadata) {
 427   return AttachmentMetadata(attachment_id, record_metadata.attachment_size());
 428 }
 429
 430 }  // namespace syncer