net/disk_cache/simple/simple_backend_impl.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/disk_cache/simple/simple_backend_impl.h"
   6
   7 #include <algorithm>
   8 #include <cstdlib>
   9
  10 #if defined(OS_POSIX)
  11 #include <sys/resource.h>
  12 #endif
  13
  14 #include "base/bind.h"
  15 #include "base/callback.h"
  16 #include "base/file_util.h"
  17 #include "base/location.h"
  18 #include "base/message_loop/message_loop_proxy.h"
  19 #include "base/metrics/field_trial.h"
  20 #include "base/metrics/histogram.h"
  21 #include "base/metrics/sparse_histogram.h"
  22 #include "base/single_thread_task_runner.h"
  23 #include "base/sys_info.h"
  24 #include "base/task_runner_util.h"
  25 #include "base/threading/sequenced_worker_pool.h"
  26 #include "base/time/time.h"
  27 #include "net/base/net_errors.h"
  28 #include "net/disk_cache/backend_impl.h"
  29 #include "net/disk_cache/simple/simple_entry_format.h"
  30 #include "net/disk_cache/simple/simple_entry_impl.h"
  31 #include "net/disk_cache/simple/simple_index.h"
  32 #include "net/disk_cache/simple/simple_index_file.h"
  33 #include "net/disk_cache/simple/simple_synchronous_entry.h"
  34 #include "net/disk_cache/simple/simple_util.h"
  35
  36 using base::Closure;
  37 using base::FilePath;
  38 using base::MessageLoopProxy;
  39 using base::SequencedWorkerPool;
  40 using base::SingleThreadTaskRunner;
  41 using base::Time;
  42 using base::DirectoryExists;
  43 using file_util::CreateDirectory;
  44
  45 namespace {
  46
  47 // Maximum number of concurrent worker pool threads, which also is the limit
  48 // on concurrent IO (as we use one thread per IO request).
  49 const int kDefaultMaxWorkerThreads = 50;
  50
  51 const char kThreadNamePrefix[] = "SimpleCache";
  52
  53 // Cache size when all other size heuristics failed.
  54 const uint64 kDefaultCacheSize = 80 * 1024 * 1024;
  55
  56 // Maximum fraction of the cache that one entry can consume.
  57 const int kMaxFileRatio = 8;
  58
  59 // A global sequenced worker pool to use for launching all tasks.
  60 SequencedWorkerPool* g_sequenced_worker_pool = NULL;
  61
  62 void MaybeCreateSequencedWorkerPool() {
  63   if (!g_sequenced_worker_pool) {
  64     int max_worker_threads = kDefaultMaxWorkerThreads;
  65
  66     const std::string thread_count_field_trial =
  67         base::FieldTrialList::FindFullName("SimpleCacheMaxThreads");
  68     if (!thread_count_field_trial.empty()) {
  69       max_worker_threads =
  70           std::max(1, std::atoi(thread_count_field_trial.c_str()));
  71     }
  72
  73     g_sequenced_worker_pool = new SequencedWorkerPool(max_worker_threads,
  74                                                       kThreadNamePrefix);
  75     g_sequenced_worker_pool->AddRef();  // Leak it.
  76   }
  77 }
  78
  79 bool g_fd_limit_histogram_has_been_populated = false;
  80
  81 void MaybeHistogramFdLimit() {
  82   if (g_fd_limit_histogram_has_been_populated)
  83     return;
  84
  85   // Used in histograms; add new entries at end.
  86   enum FdLimitStatus {
  87     FD_LIMIT_STATUS_UNSUPPORTED = 0,
  88     FD_LIMIT_STATUS_FAILED      = 1,
  89     FD_LIMIT_STATUS_SUCCEEDED   = 2,
  90     FD_LIMIT_STATUS_MAX         = 3
  91   };
  92   FdLimitStatus fd_limit_status = FD_LIMIT_STATUS_UNSUPPORTED;
  93   int soft_fd_limit = 0;
  94   int hard_fd_limit = 0;
  95
  96 #if defined(OS_POSIX)
  97   struct rlimit nofile;
  98   if (!getrlimit(RLIMIT_NOFILE, &nofile)) {
  99     soft_fd_limit = nofile.rlim_cur;
 100     hard_fd_limit = nofile.rlim_max;
 101     fd_limit_status = FD_LIMIT_STATUS_SUCCEEDED;
 102   } else {
 103     fd_limit_status = FD_LIMIT_STATUS_FAILED;
 104   }
 105 #endif
 106
 107   UMA_HISTOGRAM_ENUMERATION("SimpleCache.FileDescriptorLimitStatus",
 108                             fd_limit_status, FD_LIMIT_STATUS_MAX);
 109   if (fd_limit_status == FD_LIMIT_STATUS_SUCCEEDED) {
 110     UMA_HISTOGRAM_SPARSE_SLOWLY("SimpleCache.FileDescriptorLimitSoft",
 111                                 soft_fd_limit);
 112     UMA_HISTOGRAM_SPARSE_SLOWLY("SimpleCache.FileDescriptorLimitHard",
 113                                 hard_fd_limit);
 114   }
 115
 116   g_fd_limit_histogram_has_been_populated = true;
 117 }
 118
 119 // Must run on IO Thread.
 120 void DeleteBackendImpl(disk_cache::Backend** backend,
 121                        const net::CompletionCallback& callback,
 122                        int result) {
 123   DCHECK(*backend);
 124   delete *backend;
 125   *backend = NULL;
 126   callback.Run(result);
 127 }
 128
 129 // Detects if the files in the cache directory match the current disk cache
 130 // backend type and version. If the directory contains no cache, occupies it
 131 // with the fresh structure.
 132 //
 133 // There is a convention among disk cache backends: looking at the magic in the
 134 // file "index" it should be sufficient to determine if the cache belongs to the
 135 // currently running backend. The Simple Backend stores its index in the file
 136 // "the-real-index" (see simple_index.cc) and the file "index" only signifies
 137 // presence of the implementation's magic and version. There are two reasons for
 138 // that:
 139 // 1. Absence of the index is itself not a fatal error in the Simple Backend
 140 // 2. The Simple Backend has pickled file format for the index making it hacky
 141 //    to have the magic in the right place.
 142 bool FileStructureConsistent(const base::FilePath& path) {
 143   if (!base::PathExists(path) && !file_util::CreateDirectory(path)) {
 144     LOG(ERROR) << "Failed to create directory: " << path.LossyDisplayName();
 145     return false;
 146   }
 147   const base::FilePath fake_index = path.AppendASCII("index");
 148   base::PlatformFileError error;
 149   base::PlatformFile fake_index_file = base::CreatePlatformFile(
 150       fake_index,
 151       base::PLATFORM_FILE_OPEN | base::PLATFORM_FILE_READ,
 152       NULL,
 153       &error);
 154   if (error == base::PLATFORM_FILE_ERROR_NOT_FOUND) {
 155     base::PlatformFile file = base::CreatePlatformFile(
 156         fake_index,
 157         base::PLATFORM_FILE_CREATE | base::PLATFORM_FILE_WRITE,
 158         NULL, &error);
 159     disk_cache::SimpleFileHeader file_contents;
 160     file_contents.initial_magic_number = disk_cache::kSimpleInitialMagicNumber;
 161     file_contents.version = disk_cache::kSimpleVersion;
 162     int bytes_written = base::WritePlatformFile(
 163         file, 0, reinterpret_cast<char*>(&file_contents),
 164         sizeof(file_contents));
 165     if (!base::ClosePlatformFile(file) ||
 166         bytes_written != sizeof(file_contents)) {
 167       LOG(ERROR) << "Failed to write cache structure file: "
 168                  << path.LossyDisplayName();
 169       return false;
 170     }
 171     return true;
 172   } else if (error != base::PLATFORM_FILE_OK) {
 173     LOG(ERROR) << "Could not open cache structure file: "
 174                << path.LossyDisplayName();
 175     return false;
 176   } else {
 177     disk_cache::SimpleFileHeader file_header;
 178     int bytes_read = base::ReadPlatformFile(
 179         fake_index_file, 0, reinterpret_cast<char*>(&file_header),
 180         sizeof(file_header));
 181     if (!base::ClosePlatformFile(fake_index_file) ||
 182         bytes_read != sizeof(file_header) ||
 183         file_header.initial_magic_number !=
 184             disk_cache::kSimpleInitialMagicNumber ||
 185         file_header.version != disk_cache::kSimpleVersion) {
 186       LOG(ERROR) << "File structure does not match the disk cache backend.";
 187       return false;
 188     }
 189     return true;
 190   }
 191 }
 192
 193 void CallCompletionCallback(const net::CompletionCallback& callback,
 194                             int error_code) {
 195   DCHECK(!callback.is_null());
 196   callback.Run(error_code);
 197 }
 198
 199 void RecordIndexLoad(base::TimeTicks constructed_since, int result) {
 200   const base::TimeDelta creation_to_index = base::TimeTicks::Now() -
 201                                             constructed_since;
 202   if (result == net::OK)
 203     UMA_HISTOGRAM_TIMES("SimpleCache.CreationToIndex", creation_to_index);
 204   else
 205     UMA_HISTOGRAM_TIMES("SimpleCache.CreationToIndexFail", creation_to_index);
 206 }
 207
 208 }  // namespace
 209
 210 namespace disk_cache {
 211
 212 SimpleBackendImpl::SimpleBackendImpl(const FilePath& path,
 213                                      int max_bytes,
 214                                      net::CacheType type,
 215                                      base::SingleThreadTaskRunner* cache_thread,
 216                                      net::NetLog* net_log)
 217     : path_(path),
 218       cache_thread_(cache_thread),
 219       orig_max_size_(max_bytes),
 220       entry_operations_mode_(
 221           type == net::DISK_CACHE ?
 222               SimpleEntryImpl::OPTIMISTIC_OPERATIONS :
 223               SimpleEntryImpl::NON_OPTIMISTIC_OPERATIONS),
 224       net_log_(net_log) {
 225   MaybeHistogramFdLimit();
 226 }
 227
 228 SimpleBackendImpl::~SimpleBackendImpl() {
 229   index_->WriteToDisk();
 230 }
 231
 232 int SimpleBackendImpl::Init(const CompletionCallback& completion_callback) {
 233   MaybeCreateSequencedWorkerPool();
 234
 235   worker_pool_ = g_sequenced_worker_pool->GetTaskRunnerWithShutdownBehavior(
 236       SequencedWorkerPool::CONTINUE_ON_SHUTDOWN);
 237
 238   index_.reset(
 239       new SimpleIndex(MessageLoopProxy::current().get(),
 240                       path_,
 241                       make_scoped_ptr(new SimpleIndexFile(
 242                           cache_thread_.get(), worker_pool_.get(), path_))));
 243   index_->ExecuteWhenReady(base::Bind(&RecordIndexLoad,
 244                                       base::TimeTicks::Now()));
 245
 246   PostTaskAndReplyWithResult(
 247       cache_thread_,
 248       FROM_HERE,
 249       base::Bind(&SimpleBackendImpl::InitCacheStructureOnDisk, path_,
 250                  orig_max_size_),
 251       base::Bind(&SimpleBackendImpl::InitializeIndex, AsWeakPtr(),
 252                  completion_callback));
 253   return net::ERR_IO_PENDING;
 254 }
 255
 256 bool SimpleBackendImpl::SetMaxSize(int max_bytes) {
 257   orig_max_size_ = max_bytes;
 258   return index_->SetMaxSize(max_bytes);
 259 }
 260
 261 int SimpleBackendImpl::GetMaxFileSize() const {
 262   return index_->max_size() / kMaxFileRatio;
 263 }
 264
 265 void SimpleBackendImpl::OnDeactivated(const SimpleEntryImpl* entry) {
 266   active_entries_.erase(entry->entry_hash());
 267 }
 268
 269 net::CacheType SimpleBackendImpl::GetCacheType() const {
 270   return net::DISK_CACHE;
 271 }
 272
 273 int32 SimpleBackendImpl::GetEntryCount() const {
 274   // TODO(pasko): Use directory file count when index is not ready.
 275   return index_->GetEntryCount();
 276 }
 277
 278 int SimpleBackendImpl::OpenEntry(const std::string& key,
 279                                  Entry** entry,
 280                                  const CompletionCallback& callback) {
 281   scoped_refptr<SimpleEntryImpl> simple_entry = CreateOrFindActiveEntry(key);
 282   CompletionCallback backend_callback =
 283       base::Bind(&SimpleBackendImpl::OnEntryOpenedFromKey,
 284                  AsWeakPtr(),
 285                  key,
 286                  entry,
 287                  simple_entry,
 288                  callback);
 289   return simple_entry->OpenEntry(entry, backend_callback);
 290 }
 291
 292 int SimpleBackendImpl::CreateEntry(const std::string& key,
 293                                    Entry** entry,
 294                                    const CompletionCallback& callback) {
 295   DCHECK(key.size() > 0);
 296   scoped_refptr<SimpleEntryImpl> simple_entry = CreateOrFindActiveEntry(key);
 297   return simple_entry->CreateEntry(entry, callback);
 298 }
 299
 300 int SimpleBackendImpl::DoomEntry(const std::string& key,
 301                                  const net::CompletionCallback& callback) {
 302   scoped_refptr<SimpleEntryImpl> simple_entry = CreateOrFindActiveEntry(key);
 303   return simple_entry->DoomEntry(callback);
 304 }
 305
 306 int SimpleBackendImpl::DoomAllEntries(const CompletionCallback& callback) {
 307   return DoomEntriesBetween(Time(), Time(), callback);
 308 }
 309
 310 void SimpleBackendImpl::IndexReadyForDoom(Time initial_time,
 311                                           Time end_time,
 312                                           const CompletionCallback& callback,
 313                                           int result) {
 314   if (result != net::OK) {
 315     callback.Run(result);
 316     return;
 317   }
 318   scoped_ptr<std::vector<uint64> > removed_key_hashes(
 319       index_->RemoveEntriesBetween(initial_time, end_time).release());
 320
 321   // If any of the entries we are dooming are currently open, we need to remove
 322   // them from |active_entries_|, so that attempts to create new entries will
 323   // succeed and attempts to open them will fail.
 324   for (int i = removed_key_hashes->size() - 1; i >= 0; --i) {
 325     const uint64 entry_hash = (*removed_key_hashes)[i];
 326     EntryMap::iterator it = active_entries_.find(entry_hash);
 327     if (it == active_entries_.end())
 328       continue;
 329     SimpleEntryImpl* entry = it->second.get();
 330     entry->Doom();
 331
 332     (*removed_key_hashes)[i] = removed_key_hashes->back();
 333     removed_key_hashes->resize(removed_key_hashes->size() - 1);
 334   }
 335
 336   PostTaskAndReplyWithResult(
 337       worker_pool_, FROM_HERE,
 338       base::Bind(&SimpleSynchronousEntry::DoomEntrySet,
 339                  base::Passed(&removed_key_hashes), path_),
 340       base::Bind(&CallCompletionCallback, callback));
 341 }
 342
 343 int SimpleBackendImpl::DoomEntriesBetween(
 344     const Time initial_time,
 345     const Time end_time,
 346     const CompletionCallback& callback) {
 347   return index_->ExecuteWhenReady(
 348       base::Bind(&SimpleBackendImpl::IndexReadyForDoom, AsWeakPtr(),
 349                  initial_time, end_time, callback));
 350 }
 351
 352 int SimpleBackendImpl::DoomEntriesSince(
 353     const Time initial_time,
 354     const CompletionCallback& callback) {
 355   return DoomEntriesBetween(initial_time, Time(), callback);
 356 }
 357
 358 int SimpleBackendImpl::OpenNextEntry(void** iter,
 359                                      Entry** next_entry,
 360                                      const CompletionCallback& callback) {
 361   CompletionCallback get_next_entry =
 362       base::Bind(&SimpleBackendImpl::GetNextEntryInIterator, AsWeakPtr(), iter,
 363                  next_entry, callback);
 364   return index_->ExecuteWhenReady(get_next_entry);
 365 }
 366
 367 void SimpleBackendImpl::EndEnumeration(void** iter) {
 368   SimpleIndex::HashList* entry_list =
 369       static_cast<SimpleIndex::HashList*>(*iter);
 370   delete entry_list;
 371   *iter = NULL;
 372 }
 373
 374 void SimpleBackendImpl::GetStats(
 375     std::vector<std::pair<std::string, std::string> >* stats) {
 376   std::pair<std::string, std::string> item;
 377   item.first = "Cache type";
 378   item.second = "Simple Cache";
 379   stats->push_back(item);
 380 }
 381
 382 void SimpleBackendImpl::OnExternalCacheHit(const std::string& key) {
 383   index_->UseIfExists(key);
 384 }
 385
 386 void SimpleBackendImpl::InitializeIndex(const CompletionCallback& callback,
 387                                         const DiskStatResult& result) {
 388   if (result.net_error == net::OK) {
 389     index_->SetMaxSize(result.max_size);
 390     index_->Initialize(result.cache_dir_mtime);
 391   }
 392   callback.Run(result.net_error);
 393 }
 394
 395 SimpleBackendImpl::DiskStatResult SimpleBackendImpl::InitCacheStructureOnDisk(
 396     const base::FilePath& path,
 397     uint64 suggested_max_size) {
 398   DiskStatResult result;
 399   result.max_size = suggested_max_size;
 400   result.net_error = net::OK;
 401   if (!FileStructureConsistent(path)) {
 402     LOG(ERROR) << "Simple Cache Backend: wrong file structure on disk: "
 403                << path.LossyDisplayName();
 404     result.net_error = net::ERR_FAILED;
 405   } else {
 406     bool mtime_result =
 407         disk_cache::simple_util::GetMTime(path, &result.cache_dir_mtime);
 408     DCHECK(mtime_result);
 409     if (!result.max_size) {
 410       int64 available = base::SysInfo::AmountOfFreeDiskSpace(path);
 411       if (available < 0)
 412         result.max_size = kDefaultCacheSize;
 413       else
 414         // TODO(pasko): Move PreferedCacheSize() to cache_util.h. Also fix the
 415         // spelling.
 416         result.max_size = disk_cache::PreferedCacheSize(available);
 417     }
 418     DCHECK(result.max_size);
 419   }
 420   return result;
 421 }
 422
 423 scoped_refptr<SimpleEntryImpl> SimpleBackendImpl::CreateOrFindActiveEntry(
 424     const std::string& key) {
 425   const uint64 entry_hash = simple_util::GetEntryHashKey(key);
 426
 427   std::pair<EntryMap::iterator, bool> insert_result =
 428       active_entries_.insert(std::make_pair(entry_hash,
 429                                             base::WeakPtr<SimpleEntryImpl>()));
 430   EntryMap::iterator& it = insert_result.first;
 431   if (insert_result.second)
 432     DCHECK(!it->second.get());
 433   if (!it->second.get()) {
 434     SimpleEntryImpl* entry = new SimpleEntryImpl(
 435         path_, entry_hash, entry_operations_mode_, this, net_log_);
 436     entry->SetKey(key);
 437     it->second = entry->AsWeakPtr();
 438   }
 439   DCHECK(it->second.get());
 440   // It's possible, but unlikely, that we have an entry hash collision with a
 441   // currently active entry.
 442   if (key != it->second->key()) {
 443     it->second->Doom();
 444     DCHECK_EQ(0U, active_entries_.count(entry_hash));
 445     return CreateOrFindActiveEntry(key);
 446   }
 447   return make_scoped_refptr(it->second.get());
 448 }
 449
 450 int SimpleBackendImpl::OpenEntryFromHash(uint64 hash,
 451                                          Entry** entry,
 452                                          const CompletionCallback& callback) {
 453   EntryMap::iterator has_active = active_entries_.find(hash);
 454   if (has_active != active_entries_.end())
 455     return OpenEntry(has_active->second->key(), entry, callback);
 456
 457   scoped_refptr<SimpleEntryImpl> simple_entry =
 458       new SimpleEntryImpl(path_, hash, entry_operations_mode_, this, net_log_);
 459   CompletionCallback backend_callback =
 460       base::Bind(&SimpleBackendImpl::OnEntryOpenedFromHash,
 461                  AsWeakPtr(),
 462                  hash, entry, simple_entry, callback);
 463   return simple_entry->OpenEntry(entry, backend_callback);
 464 }
 465
 466 void SimpleBackendImpl::GetNextEntryInIterator(
 467     void** iter,
 468     Entry** next_entry,
 469     const CompletionCallback& callback,
 470     int error_code) {
 471   if (error_code != net::OK) {
 472     CallCompletionCallback(callback, error_code);
 473     return;
 474   }
 475   if (*iter == NULL) {
 476     *iter = index()->GetAllHashes().release();
 477   }
 478   SimpleIndex::HashList* entry_list =
 479       static_cast<SimpleIndex::HashList*>(*iter);
 480   while (entry_list->size() > 0) {
 481     uint64 entry_hash = entry_list->back();
 482     entry_list->pop_back();
 483     if (index()->Has(entry_hash)) {
 484       *next_entry = NULL;
 485       CompletionCallback continue_iteration = base::Bind(
 486           &SimpleBackendImpl::CheckIterationReturnValue,
 487           AsWeakPtr(),
 488           iter,
 489           next_entry,
 490           callback);
 491       int error_code_open = OpenEntryFromHash(entry_hash,
 492                                               next_entry,
 493                                               continue_iteration);
 494       if (error_code_open == net::ERR_IO_PENDING)
 495         return;
 496       if (error_code_open != net::ERR_FAILED) {
 497         CallCompletionCallback(callback, error_code_open);
 498         return;
 499       }
 500     }
 501   }
 502   CallCompletionCallback(callback, net::ERR_FAILED);
 503 }
 504
 505 void SimpleBackendImpl::OnEntryOpenedFromHash(
 506     uint64 hash,
 507     Entry** entry,
 508     scoped_refptr<SimpleEntryImpl> simple_entry,
 509     const CompletionCallback& callback,
 510     int error_code) {
 511   if (error_code != net::OK) {
 512     CallCompletionCallback(callback, error_code);
 513     return;
 514   }
 515   DCHECK(*entry);
 516   std::pair<EntryMap::iterator, bool> insert_result =
 517       active_entries_.insert(std::make_pair(hash,
 518                                             base::WeakPtr<SimpleEntryImpl>()));
 519   EntryMap::iterator& it = insert_result.first;
 520   const bool did_insert = insert_result.second;
 521   if (did_insert) {
 522     // There is no active entry corresponding to this hash. The entry created
 523     // is put in the map of active entries and returned to the caller.
 524     it->second = simple_entry->AsWeakPtr();
 525     CallCompletionCallback(callback, error_code);
 526   } else {
 527     // The entry was made active with the key while the creation from hash
 528     // occurred. The entry created from hash needs to be closed, and the one
 529     // coming from the key returned to the caller.
 530     simple_entry->Close();
 531     it->second->OpenEntry(entry, callback);
 532   }
 533 }
 534
 535 void SimpleBackendImpl::OnEntryOpenedFromKey(
 536     const std::string key,
 537     Entry** entry,
 538     scoped_refptr<SimpleEntryImpl> simple_entry,
 539     const CompletionCallback& callback,
 540     int error_code) {
 541   int final_code = error_code;
 542   if (final_code == net::OK) {
 543     bool key_matches = key.compare(simple_entry->key()) == 0;
 544     if (!key_matches) {
 545       // TODO(clamy): Add a unit test to check this code path.
 546       DLOG(WARNING) << "Key mismatch on open.";
 547       simple_entry->Doom();
 548       simple_entry->Close();
 549       final_code = net::ERR_FAILED;
 550     } else {
 551       DCHECK_EQ(simple_entry->entry_hash(), simple_util::GetEntryHashKey(key));
 552     }
 553     UMA_HISTOGRAM_BOOLEAN("SimpleCache.KeyMatchedOnOpen", key_matches);
 554   }
 555   CallCompletionCallback(callback, final_code);
 556 }
 557
 558 void SimpleBackendImpl::CheckIterationReturnValue(
 559     void** iter,
 560     Entry** entry,
 561     const CompletionCallback& callback,
 562     int error_code) {
 563   if (error_code == net::ERR_FAILED) {
 564     OpenNextEntry(iter, entry, callback);
 565     return;
 566   }
 567   CallCompletionCallback(callback, error_code);
 568 }
 569
 570 }  // namespace disk_cache