sync/internal_api/attachments/on_disk_attachment_store.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "sync/internal_api/public/attachments/on_disk_attachment_store.h"
   6
   7 #include "base/bind.h"
   8 #include "base/callback.h"
   9 #include "base/location.h"
  10 #include "base/memory/scoped_ptr.h"
  11 #include "base/metrics/histogram.h"
  12 #include "base/sequenced_task_runner.h"
  13 #include "sync/internal_api/attachments/proto/attachment_store.pb.h"
  14 #include "sync/internal_api/public/attachments/attachment_util.h"
  15 #include "sync/protocol/attachments.pb.h"
  16 #include "third_party/leveldatabase/env_chromium.h"
  17 #include "third_party/leveldatabase/src/include/leveldb/db.h"
  18 #include "third_party/leveldatabase/src/include/leveldb/options.h"
  19 #include "third_party/leveldatabase/src/include/leveldb/slice.h"
  20 #include "third_party/leveldatabase/src/include/leveldb/status.h"
  21 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
  22
  23 namespace syncer {
  24
  25 namespace {
  26
  27 // Prefix for records containing attachment data.
  28 const char kDataPrefix[] = "data-";
  29
  30 // Prefix for records containing attachment metadata.
  31 const char kMetadataPrefix[] = "metadata-";
  32
  33 const char kDatabaseMetadataKey[] = "database-metadata";
  34
  35 const int32 kCurrentSchemaVersion = 1;
  36
  37 const base::FilePath::CharType kLeveldbDirectory[] =
  38     FILE_PATH_LITERAL("leveldb");
  39
  40 // Converts syncer::AttachmentStore::Component values into
  41 // attachment_store_pb::RecordMetadata::Component.
  42 attachment_store_pb::RecordMetadata::Component ComponentToProto(
  43     syncer::AttachmentStore::Component component) {
  44   switch (component) {
  45     case AttachmentStore::MODEL_TYPE:
  46       return attachment_store_pb::RecordMetadata::MODEL_TYPE;
  47     case AttachmentStore::SYNC:
  48       return attachment_store_pb::RecordMetadata::SYNC;
  49   }
  50   NOTREACHED();
  51   return attachment_store_pb::RecordMetadata::UNKNOWN;
  52 }
  53
  54 leveldb::WriteOptions MakeWriteOptions() {
  55   leveldb::WriteOptions write_options;
  56   write_options.sync = true;
  57   return write_options;
  58 }
  59
  60 leveldb::ReadOptions MakeNonCachingReadOptions() {
  61   leveldb::ReadOptions read_options;
  62   read_options.fill_cache = false;
  63   read_options.verify_checksums = true;
  64   return read_options;
  65 }
  66
  67 leveldb::ReadOptions MakeCachingReadOptions() {
  68   leveldb::ReadOptions read_options;
  69   read_options.fill_cache = true;
  70   read_options.verify_checksums = true;
  71   return read_options;
  72 }
  73
  74 leveldb::Status ReadStoreMetadata(
  75     leveldb::DB* db,
  76     attachment_store_pb::StoreMetadata* metadata) {
  77   std::string data_str;
  78
  79   leveldb::Status status =
  80       db->Get(MakeCachingReadOptions(), kDatabaseMetadataKey, &data_str);
  81   if (!status.ok())
  82     return status;
  83   if (!metadata->ParseFromString(data_str))
  84     return leveldb::Status::Corruption("Metadata record corruption");
  85   return leveldb::Status::OK();
  86 }
  87
  88 leveldb::Status WriteStoreMetadata(
  89     leveldb::DB* db,
  90     const attachment_store_pb::StoreMetadata& metadata) {
  91   std::string data_str;
  92
  93   metadata.SerializeToString(&data_str);
  94   return db->Put(MakeWriteOptions(), kDatabaseMetadataKey, data_str);
  95 }
  96
  97 // Adds reference to component into RecordMetadata::component set.
  98 // Returns true if record_metadata was modified and needs to be written to disk.
  99 bool SetReferenceInRecordMetadata(
 100     attachment_store_pb::RecordMetadata* record_metadata,
 101     attachment_store_pb::RecordMetadata::Component proto_component) {
 102   DCHECK(record_metadata);
 103   for (const int component : record_metadata->component()) {
 104     if (component == proto_component)
 105       return false;
 106   }
 107   record_metadata->add_component(proto_component);
 108   return true;
 109 }
 110
 111 // Drops reference to component from RecordMetadata::component set.
 112 // Returns true if record_metadata was modified and needs to be written to disk.
 113 bool DropReferenceInRecordMetadata(
 114     attachment_store_pb::RecordMetadata* record_metadata,
 115     attachment_store_pb::RecordMetadata::Component proto_component) {
 116   DCHECK(record_metadata);
 117   bool component_removed = false;
 118   ::google::protobuf::RepeatedField<int>* mutable_components =
 119       record_metadata->mutable_component();
 120   for (int i = 0; i < mutable_components->size();) {
 121     if (mutable_components->Get(i) == proto_component) {
 122       if (i < mutable_components->size() - 1) {
 123         // Don't swap last element with itself.
 124         mutable_components->SwapElements(i, mutable_components->size() - 1);
 125       }
 126       mutable_components->RemoveLast();
 127       component_removed = true;
 128     } else {
 129       ++i;
 130     }
 131   }
 132   return component_removed;
 133 }
 134
 135 bool AttachmentHasReferenceFromComponent(
 136     const attachment_store_pb::RecordMetadata& record_metadata,
 137     attachment_store_pb::RecordMetadata::Component proto_component) {
 138   for (const auto& reference_component : record_metadata.component()) {
 139     if (reference_component == proto_component) {
 140       return true;
 141     }
 142   }
 143   return false;
 144 }
 145
 146 }  // namespace
 147
 148 OnDiskAttachmentStore::OnDiskAttachmentStore(
 149     const scoped_refptr<base::SequencedTaskRunner>& callback_task_runner,
 150     const base::FilePath& path)
 151     : AttachmentStoreBackend(callback_task_runner), path_(path) {
 152 }
 153
 154 OnDiskAttachmentStore::~OnDiskAttachmentStore() {
 155 }
 156
 157 void OnDiskAttachmentStore::Init(
 158     const AttachmentStore::InitCallback& callback) {
 159   DCHECK(CalledOnValidThread());
 160   AttachmentStore::Result result_code = OpenOrCreate(path_);
 161   UMA_HISTOGRAM_ENUMERATION("Sync.Attachments.StoreInitResult", result_code,
 162                             AttachmentStore::RESULT_SIZE);
 163   PostCallback(base::Bind(callback, result_code));
 164 }
 165
 166 void OnDiskAttachmentStore::Read(
 167     AttachmentStore::Component component,
 168     const AttachmentIdList& ids,
 169     const AttachmentStore::ReadCallback& callback) {
 170   DCHECK(CalledOnValidThread());
 171   scoped_ptr<AttachmentMap> result_map(new AttachmentMap());
 172   scoped_ptr<AttachmentIdList> unavailable_attachments(new AttachmentIdList());
 173
 174   AttachmentStore::Result result_code =
 175       AttachmentStore::STORE_INITIALIZATION_FAILED;
 176
 177   if (db_) {
 178     result_code = AttachmentStore::SUCCESS;
 179     for (const auto& id : ids) {
 180       scoped_ptr<Attachment> attachment;
 181       attachment = ReadSingleAttachment(id, component);
 182       if (attachment) {
 183         result_map->insert(std::make_pair(id, *attachment));
 184       } else {
 185         unavailable_attachments->push_back(id);
 186       }
 187     }
 188     result_code = unavailable_attachments->empty()
 189                       ? AttachmentStore::SUCCESS
 190                       : AttachmentStore::UNSPECIFIED_ERROR;
 191   } else {
 192     *unavailable_attachments = ids;
 193   }
 194
 195   PostCallback(base::Bind(callback, result_code, base::Passed(&result_map),
 196                           base::Passed(&unavailable_attachments)));
 197 }
 198
 199 void OnDiskAttachmentStore::Write(
 200     AttachmentStore::Component component,
 201     const AttachmentList& attachments,
 202     const AttachmentStore::WriteCallback& callback) {
 203   DCHECK(CalledOnValidThread());
 204   AttachmentStore::Result result_code =
 205       AttachmentStore::STORE_INITIALIZATION_FAILED;
 206
 207   if (db_) {
 208     result_code = AttachmentStore::SUCCESS;
 209     AttachmentList::const_iterator iter = attachments.begin();
 210     const AttachmentList::const_iterator end = attachments.end();
 211     for (; iter != end; ++iter) {
 212       if (!WriteSingleAttachment(*iter, component))
 213         result_code = AttachmentStore::UNSPECIFIED_ERROR;
 214     }
 215   }
 216   PostCallback(base::Bind(callback, result_code));
 217 }
 218
 219 void OnDiskAttachmentStore::SetReference(AttachmentStore::Component component,
 220                                          const AttachmentIdList& ids) {
 221   DCHECK(CalledOnValidThread());
 222   if (!db_)
 223     return;
 224   attachment_store_pb::RecordMetadata::Component proto_component =
 225       ComponentToProto(component);
 226   for (const auto& id : ids) {
 227     attachment_store_pb::RecordMetadata record_metadata;
 228     if (!ReadSingleRecordMetadata(id, &record_metadata))
 229       continue;
 230     if (SetReferenceInRecordMetadata(&record_metadata, proto_component))
 231       WriteSingleRecordMetadata(id, record_metadata);
 232   }
 233 }
 234
 235 void OnDiskAttachmentStore::DropReference(
 236     AttachmentStore::Component component,
 237     const AttachmentIdList& ids,
 238     const AttachmentStore::DropCallback& callback) {
 239   DCHECK(CalledOnValidThread());
 240   AttachmentStore::Result result_code =
 241       AttachmentStore::STORE_INITIALIZATION_FAILED;
 242   if (db_) {
 243     attachment_store_pb::RecordMetadata::Component proto_component =
 244         ComponentToProto(component);
 245     result_code = AttachmentStore::SUCCESS;
 246     leveldb::WriteOptions write_options = MakeWriteOptions();
 247     for (const auto& id : ids) {
 248       attachment_store_pb::RecordMetadata record_metadata;
 249       if (!ReadSingleRecordMetadata(id, &record_metadata))
 250         continue;  // Record not found.
 251       if (!DropReferenceInRecordMetadata(&record_metadata, proto_component))
 252         continue;  // Component is not in components set. Metadata was not
 253                    // updated.
 254       if (record_metadata.component_size() == 0) {
 255         // Last reference dropped. Need to delete attachment.
 256         leveldb::WriteBatch write_batch;
 257         write_batch.Delete(MakeDataKeyFromAttachmentId(id));
 258         write_batch.Delete(MakeMetadataKeyFromAttachmentId(id));
 259
 260         leveldb::Status status = db_->Write(write_options, &write_batch);
 261         if (!status.ok()) {
 262           // DB::Delete doesn't check if record exists, it returns ok just like
 263           // AttachmentStore::Drop should.
 264           DVLOG(1) << "DB::Write failed: status=" << status.ToString();
 265           result_code = AttachmentStore::UNSPECIFIED_ERROR;
 266         }
 267       } else {
 268         WriteSingleRecordMetadata(id, record_metadata);
 269       }
 270     }
 271   }
 272   PostCallback(base::Bind(callback, result_code));
 273 }
 274
 275 void OnDiskAttachmentStore::ReadMetadataById(
 276     AttachmentStore::Component component,
 277     const AttachmentIdList& ids,
 278     const AttachmentStore::ReadMetadataCallback& callback) {
 279   DCHECK(CalledOnValidThread());
 280   AttachmentStore::Result result_code =
 281       AttachmentStore::STORE_INITIALIZATION_FAILED;
 282   scoped_ptr<AttachmentMetadataList> metadata_list(
 283       new AttachmentMetadataList());
 284   if (db_) {
 285     result_code = AttachmentStore::SUCCESS;
 286     for (const auto& id : ids) {
 287       attachment_store_pb::RecordMetadata record_metadata;
 288       if (!ReadSingleRecordMetadata(id, &record_metadata)) {
 289         result_code = AttachmentStore::UNSPECIFIED_ERROR;
 290         continue;
 291       }
 292       if (!AttachmentHasReferenceFromComponent(record_metadata,
 293                                                ComponentToProto(component))) {
 294         result_code = AttachmentStore::UNSPECIFIED_ERROR;
 295         continue;
 296       }
 297       metadata_list->push_back(MakeAttachmentMetadata(id, record_metadata));
 298     }
 299   }
 300   PostCallback(base::Bind(callback, result_code, base::Passed(&metadata_list)));
 301 }
 302
 303 void OnDiskAttachmentStore::ReadMetadata(
 304     AttachmentStore::Component component,
 305     const AttachmentStore::ReadMetadataCallback& callback) {
 306   DCHECK(CalledOnValidThread());
 307   AttachmentStore::Result result_code =
 308       AttachmentStore::STORE_INITIALIZATION_FAILED;
 309   scoped_ptr<AttachmentMetadataList> metadata_list(
 310       new AttachmentMetadataList());
 311
 312   if (db_) {
 313     attachment_store_pb::RecordMetadata::Component proto_component =
 314         ComponentToProto(component);
 315     result_code = AttachmentStore::SUCCESS;
 316     scoped_ptr<leveldb::Iterator> db_iterator(
 317         db_->NewIterator(MakeNonCachingReadOptions()));
 318     DCHECK(db_iterator);
 319     for (db_iterator->Seek(kMetadataPrefix); db_iterator->Valid();
 320          db_iterator->Next()) {
 321       leveldb::Slice key = db_iterator->key();
 322       if (!key.starts_with(kMetadataPrefix)) {
 323         break;
 324       }
 325       // Make AttachmentId from levelDB key.
 326       key.remove_prefix(strlen(kMetadataPrefix));
 327       sync_pb::AttachmentIdProto id_proto;
 328       id_proto.set_unique_id(key.ToString());
 329       AttachmentId id = AttachmentId::CreateFromProto(id_proto);
 330       // Parse metadata record.
 331       attachment_store_pb::RecordMetadata record_metadata;
 332       if (!record_metadata.ParseFromString(db_iterator->value().ToString())) {
 333         DVLOG(1) << "RecordMetadata::ParseFromString failed";
 334         result_code = AttachmentStore::UNSPECIFIED_ERROR;
 335         continue;
 336       }
 337       DCHECK(record_metadata.component_size() > 0);
 338       if (AttachmentHasReferenceFromComponent(record_metadata, proto_component))
 339         metadata_list->push_back(MakeAttachmentMetadata(id, record_metadata));
 340     }
 341
 342     if (!db_iterator->status().ok()) {
 343       DVLOG(1) << "DB Iterator failed: status="
 344                << db_iterator->status().ToString();
 345       result_code = AttachmentStore::UNSPECIFIED_ERROR;
 346     }
 347   }
 348
 349   PostCallback(base::Bind(callback, result_code, base::Passed(&metadata_list)));
 350 }
 351
 352 AttachmentStore::Result OnDiskAttachmentStore::OpenOrCreate(
 353     const base::FilePath& path) {
 354   DCHECK(!db_);
 355   base::FilePath leveldb_path = path.Append(kLeveldbDirectory);
 356
 357   leveldb::DB* db_raw;
 358   scoped_ptr<leveldb::DB> db;
 359   leveldb::Options options;
 360   options.create_if_missing = true;
 361   options.reuse_logs = leveldb_env::kDefaultLogReuseOptionValue;
 362   // TODO(pavely): crbug/424287 Consider adding info_log, block_cache and
 363   // filter_policy to options.
 364   leveldb::Status status =
 365       leveldb::DB::Open(options, leveldb_path.AsUTF8Unsafe(), &db_raw);
 366   if (!status.ok()) {
 367     DVLOG(1) << "DB::Open failed: status=" << status.ToString()
 368              << ", path=" << path.AsUTF8Unsafe();
 369     return AttachmentStore::UNSPECIFIED_ERROR;
 370   }
 371
 372   db.reset(db_raw);
 373
 374   attachment_store_pb::StoreMetadata metadata;
 375   status = ReadStoreMetadata(db.get(), &metadata);
 376   if (!status.ok() && !status.IsNotFound()) {
 377     DVLOG(1) << "ReadStoreMetadata failed: status=" << status.ToString();
 378     return AttachmentStore::UNSPECIFIED_ERROR;
 379   }
 380   if (status.IsNotFound()) {
 381     // Brand new database.
 382     metadata.set_schema_version(kCurrentSchemaVersion);
 383     status = WriteStoreMetadata(db.get(), metadata);
 384     if (!status.ok()) {
 385       DVLOG(1) << "WriteStoreMetadata failed: status=" << status.ToString();
 386       return AttachmentStore::UNSPECIFIED_ERROR;
 387     }
 388   }
 389   DCHECK(status.ok());
 390
 391   // Upgrade code goes here.
 392
 393   if (metadata.schema_version() != kCurrentSchemaVersion) {
 394     DVLOG(1) << "Unknown schema version: " << metadata.schema_version();
 395     return AttachmentStore::UNSPECIFIED_ERROR;
 396   }
 397
 398   db_ = db.Pass();
 399   return AttachmentStore::SUCCESS;
 400 }
 401
 402 scoped_ptr<Attachment> OnDiskAttachmentStore::ReadSingleAttachment(
 403     const AttachmentId& attachment_id,
 404     AttachmentStore::Component component) {
 405   scoped_ptr<Attachment> attachment;
 406   attachment_store_pb::RecordMetadata record_metadata;
 407   if (!ReadSingleRecordMetadata(attachment_id, &record_metadata)) {
 408     return attachment.Pass();
 409   }
 410   if (!AttachmentHasReferenceFromComponent(record_metadata,
 411                                            ComponentToProto(component)))
 412     return attachment.Pass();
 413
 414   const std::string key = MakeDataKeyFromAttachmentId(attachment_id);
 415   std::string data_str;
 416   leveldb::Status status = db_->Get(
 417       MakeNonCachingReadOptions(), key, &data_str);
 418   if (!status.ok()) {
 419     DVLOG(1) << "DB::Get for data failed: status=" << status.ToString();
 420     return attachment.Pass();
 421   }
 422   scoped_refptr<base::RefCountedMemory> data =
 423       base::RefCountedString::TakeString(&data_str);
 424   uint32_t crc32c = ComputeCrc32c(data);
 425   if (record_metadata.has_crc32c()) {
 426     if (record_metadata.crc32c() != crc32c) {
 427       DVLOG(1) << "Attachment crc32c does not match value read from store";
 428       return attachment.Pass();
 429     }
 430     if (record_metadata.crc32c() != attachment_id.GetCrc32c()) {
 431       DVLOG(1) << "Attachment crc32c does not match value in AttachmentId";
 432       return attachment.Pass();
 433     }
 434   }
 435   attachment.reset(
 436       new Attachment(Attachment::CreateFromParts(attachment_id, data)));
 437   return attachment.Pass();
 438 }
 439
 440 bool OnDiskAttachmentStore::WriteSingleAttachment(
 441     const Attachment& attachment,
 442     AttachmentStore::Component component) {
 443   const std::string metadata_key =
 444       MakeMetadataKeyFromAttachmentId(attachment.GetId());
 445   const std::string data_key = MakeDataKeyFromAttachmentId(attachment.GetId());
 446
 447   std::string metadata_str;
 448   leveldb::Status status =
 449       db_->Get(MakeCachingReadOptions(), metadata_key, &metadata_str);
 450   if (status.ok()) {
 451     // Entry exists, don't overwrite.
 452     return true;
 453   } else if (!status.IsNotFound()) {
 454     // Entry exists but failed to read.
 455     DVLOG(1) << "DB::Get failed: status=" << status.ToString();
 456     return false;
 457   }
 458   DCHECK(status.IsNotFound());
 459
 460   leveldb::WriteBatch write_batch;
 461   // Write metadata.
 462   attachment_store_pb::RecordMetadata metadata;
 463   metadata.set_attachment_size(attachment.GetData()->size());
 464   metadata.set_crc32c(attachment.GetCrc32c());
 465   SetReferenceInRecordMetadata(&metadata, ComponentToProto(component));
 466   metadata_str = metadata.SerializeAsString();
 467   write_batch.Put(metadata_key, metadata_str);
 468   // Write data.
 469   scoped_refptr<base::RefCountedMemory> data = attachment.GetData();
 470   leveldb::Slice data_slice(data->front_as<char>(), data->size());
 471   write_batch.Put(data_key, data_slice);
 472
 473   status = db_->Write(MakeWriteOptions(), &write_batch);
 474   if (!status.ok()) {
 475     // Failed to write.
 476     DVLOG(1) << "DB::Write failed: status=" << status.ToString();
 477     return false;
 478   }
 479   return true;
 480 }
 481
 482 bool OnDiskAttachmentStore::ReadSingleRecordMetadata(
 483     const AttachmentId& attachment_id,
 484     attachment_store_pb::RecordMetadata* record_metadata) {
 485   DCHECK(record_metadata);
 486   const std::string metadata_key =
 487       MakeMetadataKeyFromAttachmentId(attachment_id);
 488   std::string metadata_str;
 489   leveldb::Status status =
 490       db_->Get(MakeCachingReadOptions(), metadata_key, &metadata_str);
 491   if (!status.ok()) {
 492     DVLOG(1) << "DB::Get for metadata failed: status=" << status.ToString();
 493     return false;
 494   }
 495   if (!record_metadata->ParseFromString(metadata_str)) {
 496     DVLOG(1) << "RecordMetadata::ParseFromString failed";
 497     return false;
 498   }
 499   DCHECK(record_metadata->component_size() > 0);
 500   return true;
 501 }
 502
 503 bool OnDiskAttachmentStore::WriteSingleRecordMetadata(
 504     const AttachmentId& attachment_id,
 505     const attachment_store_pb::RecordMetadata& record_metadata) {
 506   const std::string metadata_key =
 507       MakeMetadataKeyFromAttachmentId(attachment_id);
 508   std::string metadata_str;
 509   metadata_str = record_metadata.SerializeAsString();
 510   leveldb::Status status =
 511       db_->Put(MakeWriteOptions(), metadata_key, metadata_str);
 512   if (!status.ok()) {
 513     DVLOG(1) << "DB::Put failed: status=" << status.ToString();
 514     return false;
 515   }
 516   return true;
 517 }
 518
 519 std::string OnDiskAttachmentStore::MakeDataKeyFromAttachmentId(
 520     const AttachmentId& attachment_id) {
 521   std::string key = kDataPrefix + attachment_id.GetProto().unique_id();
 522   return key;
 523 }
 524
 525 std::string OnDiskAttachmentStore::MakeMetadataKeyFromAttachmentId(
 526     const AttachmentId& attachment_id) {
 527   std::string key = kMetadataPrefix + attachment_id.GetProto().unique_id();
 528   return key;
 529 }
 530
 531 AttachmentMetadata OnDiskAttachmentStore::MakeAttachmentMetadata(
 532     const AttachmentId& attachment_id,
 533     const attachment_store_pb::RecordMetadata& record_metadata) {
 534   return AttachmentMetadata(attachment_id, record_metadata.attachment_size());
 535 }
 536
 537 }  // namespace syncer