net/disk_cache/v3/backend_worker.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/disk_cache/backend_impl.h"
   6
   7 #include "base/bind.h"
   8 #include "base/bind_helpers.h"
   9 #include "base/file_util.h"
  10 #include "base/files/file_path.h"
  11 #include "base/hash.h"
  12 #include "base/message_loop/message_loop.h"
  13 #include "base/metrics/field_trial.h"
  14 #include "base/metrics/histogram.h"
  15 #include "base/metrics/stats_counters.h"
  16 #include "base/rand_util.h"
  17 #include "base/strings/string_util.h"
  18 #include "base/strings/stringprintf.h"
  19 #include "base/sys_info.h"
  20 #include "base/threading/thread_restrictions.h"
  21 #include "base/time/time.h"
  22 #include "base/timer/timer.h"
  23 #include "net/base/net_errors.h"
  24 #include "net/disk_cache/cache_util.h"
  25 #include "net/disk_cache/entry_impl.h"
  26 #include "net/disk_cache/errors.h"
  27 #include "net/disk_cache/experiments.h"
  28 #include "net/disk_cache/file.h"
  29
  30 // This has to be defined before including histogram_macros.h from this file.
  31 #define NET_DISK_CACHE_BACKEND_IMPL_CC_
  32 #include "net/disk_cache/histogram_macros.h"
  33
  34 using base::Time;
  35 using base::TimeDelta;
  36 using base::TimeTicks;
  37
  38 namespace {
  39
  40 const char* kIndexName = "index";
  41
  42 // Seems like ~240 MB correspond to less than 50k entries for 99% of the people.
  43 // Note that the actual target is to keep the index table load factor under 55%
  44 // for most users.
  45 const int k64kEntriesStore = 240 * 1000 * 1000;
  46 const int kBaseTableLen = 64 * 1024;
  47 const int kDefaultCacheSize = 80 * 1024 * 1024;
  48
  49 // Avoid trimming the cache for the first 5 minutes (10 timer ticks).
  50 const int kTrimDelay = 10;
  51
  52 int DesiredIndexTableLen(int32 storage_size) {
  53   if (storage_size <= k64kEntriesStore)
  54     return kBaseTableLen;
  55   if (storage_size <= k64kEntriesStore * 2)
  56     return kBaseTableLen * 2;
  57   if (storage_size <= k64kEntriesStore * 4)
  58     return kBaseTableLen * 4;
  59   if (storage_size <= k64kEntriesStore * 8)
  60     return kBaseTableLen * 8;
  61
  62   // The biggest storage_size for int32 requires a 4 MB table.
  63   return kBaseTableLen * 16;
  64 }
  65
  66 int MaxStorageSizeForTable(int table_len) {
  67   return table_len * (k64kEntriesStore / kBaseTableLen);
  68 }
  69
  70 size_t GetIndexSize(int table_len) {
  71   size_t table_size = sizeof(disk_cache::CacheAddr) * table_len;
  72   return sizeof(disk_cache::IndexHeader) + table_size;
  73 }
  74
  75 // ------------------------------------------------------------------------
  76
  77 // Sets group for the current experiment. Returns false if the files should be
  78 // discarded.
  79 bool InitExperiment(disk_cache::IndexHeader* header, bool cache_created) {
  80   if (header->experiment == disk_cache::EXPERIMENT_OLD_FILE1 ||
  81       header->experiment == disk_cache::EXPERIMENT_OLD_FILE2) {
  82     // Discard current cache.
  83     return false;
  84   }
  85
  86   if (base::FieldTrialList::FindFullName("SimpleCacheTrial") ==
  87           "ExperimentControl") {
  88     if (cache_created) {
  89       header->experiment = disk_cache::EXPERIMENT_SIMPLE_CONTROL;
  90       return true;
  91     } else if (header->experiment != disk_cache::EXPERIMENT_SIMPLE_CONTROL) {
  92       return false;
  93     }
  94   }
  95
  96   header->experiment = disk_cache::NO_EXPERIMENT;
  97   return true;
  98 }
  99
 100 }  // namespace
 101
 102 // ------------------------------------------------------------------------
 103
 104 namespace disk_cache {
 105
 106 BackendImpl::BackendImpl(const base::FilePath& path,
 107                          base::MessageLoopProxy* cache_thread,
 108                          net::NetLog* net_log)
 109     : background_queue_(this, cache_thread),
 110       path_(path),
 111       block_files_(path),
 112       mask_(0),
 113       max_size_(0),
 114       up_ticks_(0),
 115       cache_type_(net::DISK_CACHE),
 116       uma_report_(0),
 117       user_flags_(0),
 118       init_(false),
 119       restarted_(false),
 120       unit_test_(false),
 121       read_only_(false),
 122       disabled_(false),
 123       new_eviction_(false),
 124       first_timer_(true),
 125       user_load_(false),
 126       net_log_(net_log),
 127       done_(true, false),
 128       ptr_factory_(this) {
 129 }
 130
 131 int BackendImpl::SyncInit() {
 132 #if defined(NET_BUILD_STRESS_CACHE)
 133   // Start evictions right away.
 134   up_ticks_ = kTrimDelay * 2;
 135 #endif
 136   DCHECK(!init_);
 137   if (init_)
 138     return net::ERR_FAILED;
 139
 140   bool create_files = false;
 141   if (!InitBackingStore(&create_files)) {
 142     ReportError(ERR_STORAGE_ERROR);
 143     return net::ERR_FAILED;
 144   }
 145
 146   num_refs_ = num_pending_io_ = max_refs_ = 0;
 147   entry_count_ = byte_count_ = 0;
 148
 149   if (!restarted_) {
 150     buffer_bytes_ = 0;
 151     trace_object_ = TraceObject::GetTraceObject();
 152     // Create a recurrent timer of 30 secs.
 153     int timer_delay = unit_test_ ? 1000 : 30000;
 154     timer_.reset(new base::RepeatingTimer<BackendImpl>());
 155     timer_->Start(FROM_HERE, TimeDelta::FromMilliseconds(timer_delay), this,
 156                   &BackendImpl::OnStatsTimer);
 157   }
 158
 159   init_ = true;
 160   Trace("Init");
 161
 162   if (data_->header.experiment != NO_EXPERIMENT &&
 163       cache_type_ != net::DISK_CACHE) {
 164     // No experiment for other caches.
 165     return net::ERR_FAILED;
 166   }
 167
 168   if (!(user_flags_ & kNoRandom)) {
 169     // The unit test controls directly what to test.
 170     new_eviction_ = (cache_type_ == net::DISK_CACHE);
 171   }
 172
 173   if (!CheckIndex()) {
 174     ReportError(ERR_INIT_FAILED);
 175     return net::ERR_FAILED;
 176   }
 177
 178   if (!restarted_ && (create_files || !data_->header.num_entries))
 179     ReportError(ERR_CACHE_CREATED);
 180
 181   if (!(user_flags_ & kNoRandom) && cache_type_ == net::DISK_CACHE &&
 182       !InitExperiment(&data_->header, create_files)) {
 183     return net::ERR_FAILED;
 184   }
 185
 186   // We don't care if the value overflows. The only thing we care about is that
 187   // the id cannot be zero, because that value is used as "not dirty".
 188   // Increasing the value once per second gives us many years before we start
 189   // having collisions.
 190   data_->header.this_id++;
 191   if (!data_->header.this_id)
 192     data_->header.this_id++;
 193
 194   bool previous_crash = (data_->header.crash != 0);
 195   data_->header.crash = 1;
 196
 197   if (!block_files_.Init(create_files))
 198     return net::ERR_FAILED;
 199
 200   // We want to minimize the changes to cache for an AppCache.
 201   if (cache_type() == net::APP_CACHE) {
 202     DCHECK(!new_eviction_);
 203     read_only_ = true;
 204   } else if (cache_type() == net::SHADER_CACHE) {
 205     DCHECK(!new_eviction_);
 206   }
 207
 208   eviction_.Init(this);
 209
 210   // stats_ and rankings_ may end up calling back to us so we better be enabled.
 211   disabled_ = false;
 212   if (!InitStats())
 213     return net::ERR_FAILED;
 214
 215   disabled_ = !rankings_.Init(this, new_eviction_);
 216
 217 #if defined(STRESS_CACHE_EXTENDED_VALIDATION)
 218   trace_object_->EnableTracing(false);
 219   int sc = SelfCheck();
 220   if (sc < 0 && sc != ERR_NUM_ENTRIES_MISMATCH)
 221     NOTREACHED();
 222   trace_object_->EnableTracing(true);
 223 #endif
 224
 225   if (previous_crash) {
 226     ReportError(ERR_PREVIOUS_CRASH);
 227   } else if (!restarted_) {
 228     ReportError(ERR_NO_ERROR);
 229   }
 230
 231   FlushIndex();
 232
 233   return disabled_ ? net::ERR_FAILED : net::OK;
 234 }
 235
 236 void BackendImpl::PrepareForRestart() {
 237   // Reset the mask_ if it was not given by the user.
 238   if (!(user_flags_ & kMask))
 239     mask_ = 0;
 240
 241   if (!(user_flags_ & kNewEviction))
 242     new_eviction_ = false;
 243
 244   disabled_ = true;
 245   data_->header.crash = 0;
 246   index_->Flush();
 247   index_ = NULL;
 248   data_ = NULL;
 249   block_files_.CloseFiles();
 250   rankings_.Reset();
 251   init_ = false;
 252   restarted_ = true;
 253 }
 254
 255 BackendImpl::~BackendImpl() {
 256   if (user_flags_ & kNoRandom) {
 257     // This is a unit test, so we want to be strict about not leaking entries
 258     // and completing all the work.
 259     background_queue_.WaitForPendingIO();
 260   } else {
 261     // This is most likely not a test, so we want to do as little work as
 262     // possible at this time, at the price of leaving dirty entries behind.
 263     background_queue_.DropPendingIO();
 264   }
 265
 266   if (background_queue_.BackgroundIsCurrentThread()) {
 267     // Unit tests may use the same thread for everything.
 268     CleanupCache();
 269   } else {
 270     background_queue_.background_thread()->PostTask(
 271         FROM_HERE, base::Bind(&FinalCleanupCallback, base::Unretained(this)));
 272     // http://crbug.com/74623
 273     base::ThreadRestrictions::ScopedAllowWait allow_wait;
 274     done_.Wait();
 275   }
 276 }
 277
 278 void BackendImpl::CleanupCache() {
 279   Trace("Backend Cleanup");
 280   eviction_.Stop();
 281   timer_.reset();
 282
 283   if (init_) {
 284     StoreStats();
 285     if (data_)
 286       data_->header.crash = 0;
 287
 288     if (user_flags_ & kNoRandom) {
 289       // This is a net_unittest, verify that we are not 'leaking' entries.
 290       File::WaitForPendingIO(&num_pending_io_);
 291       DCHECK(!num_refs_);
 292     } else {
 293       File::DropPendingIO();
 294     }
 295   }
 296   block_files_.CloseFiles();
 297   FlushIndex();
 298   index_ = NULL;
 299   ptr_factory_.InvalidateWeakPtrs();
 300   done_.Signal();
 301 }
 302
 303 base::FilePath BackendImpl::GetFileName(Addr address) const {
 304   if (!address.is_separate_file() || !address.is_initialized()) {
 305     NOTREACHED();
 306     return base::FilePath();
 307   }
 308
 309   std::string tmp = base::StringPrintf("f_%06x", address.FileNumber());
 310   return path_.AppendASCII(tmp);
 311 }
 312
 313 // We just created a new file so we're going to write the header and set the
 314 // file length to include the hash table (zero filled).
 315 bool BackendImpl::CreateBackingStore(disk_cache::File* file) {
 316   AdjustMaxCacheSize(0);
 317
 318   IndexHeader header;
 319   header.table_len = DesiredIndexTableLen(max_size_);
 320
 321   // We need file version 2.1 for the new eviction algorithm.
 322   if (new_eviction_)
 323     header.version = 0x20001;
 324
 325   header.create_time = Time::Now().ToInternalValue();
 326
 327   if (!file->Write(&header, sizeof(header), 0))
 328     return false;
 329
 330   return file->SetLength(GetIndexSize(header.table_len));
 331 }
 332
 333 bool BackendImpl::InitBackingStore(bool* file_created) {
 334   if (!file_util::CreateDirectory(path_))
 335     return false;
 336
 337   base::FilePath index_name = path_.AppendASCII(kIndexName);
 338
 339   int flags = base::PLATFORM_FILE_READ |
 340               base::PLATFORM_FILE_WRITE |
 341               base::PLATFORM_FILE_OPEN_ALWAYS |
 342               base::PLATFORM_FILE_EXCLUSIVE_WRITE;
 343   scoped_refptr<disk_cache::File> file(new disk_cache::File(
 344       base::CreatePlatformFile(index_name, flags, file_created, NULL)));
 345
 346   if (!file->IsValid())
 347     return false;
 348
 349   bool ret = true;
 350   if (*file_created)
 351     ret = CreateBackingStore(file.get());
 352
 353   file = NULL;
 354   if (!ret)
 355     return false;
 356
 357   index_ = new MappedFile();
 358   data_ = reinterpret_cast<Index*>(index_->Init(index_name, 0));
 359   if (!data_) {
 360     LOG(ERROR) << "Unable to map Index file";
 361     return false;
 362   }
 363
 364   if (index_->GetLength() < sizeof(Index)) {
 365     // We verify this again on CheckIndex() but it's easier to make sure now
 366     // that the header is there.
 367     LOG(ERROR) << "Corrupt Index file";
 368     return false;
 369   }
 370
 371   return true;
 372 }
 373
 374 void BackendImpl::ReportError(int error) {
 375   STRESS_DCHECK(!error || error == ERR_PREVIOUS_CRASH ||
 376                 error == ERR_CACHE_CREATED);
 377
 378   // We transmit positive numbers, instead of direct error codes.
 379   DCHECK_LE(error, 0);
 380   CACHE_UMA(CACHE_ERROR, "Error", 0, error * -1);
 381 }
 382
 383
 384 bool BackendImpl::CheckIndex() {
 385   DCHECK(data_);
 386
 387   size_t current_size = index_->GetLength();
 388   if (current_size < sizeof(Index)) {
 389     LOG(ERROR) << "Corrupt Index file";
 390     return false;
 391   }
 392
 393   if (new_eviction_) {
 394     // We support versions 2.0 and 2.1, upgrading 2.0 to 2.1.
 395     if (kIndexMagic != data_->header.magic ||
 396         kCurrentVersion >> 16 != data_->header.version >> 16) {
 397       LOG(ERROR) << "Invalid file version or magic";
 398       return false;
 399     }
 400     if (kCurrentVersion == data_->header.version) {
 401       // We need file version 2.1 for the new eviction algorithm.
 402       UpgradeTo2_1();
 403     }
 404   } else {
 405     if (kIndexMagic != data_->header.magic ||
 406         kCurrentVersion != data_->header.version) {
 407       LOG(ERROR) << "Invalid file version or magic";
 408       return false;
 409     }
 410   }
 411
 412   if (!data_->header.table_len) {
 413     LOG(ERROR) << "Invalid table size";
 414     return false;
 415   }
 416
 417   if (current_size < GetIndexSize(data_->header.table_len) ||
 418       data_->header.table_len & (kBaseTableLen - 1)) {
 419     LOG(ERROR) << "Corrupt Index file";
 420     return false;
 421   }
 422
 423   AdjustMaxCacheSize(data_->header.table_len);
 424
 425 #if !defined(NET_BUILD_STRESS_CACHE)
 426   if (data_->header.num_bytes < 0 ||
 427       (max_size_ < kint32max - kDefaultCacheSize &&
 428        data_->header.num_bytes > max_size_ + kDefaultCacheSize)) {
 429     LOG(ERROR) << "Invalid cache (current) size";
 430     return false;
 431   }
 432 #endif
 433
 434   if (data_->header.num_entries < 0) {
 435     LOG(ERROR) << "Invalid number of entries";
 436     return false;
 437   }
 438
 439   if (!mask_)
 440     mask_ = data_->header.table_len - 1;
 441
 442   // Load the table into memory with a single read.
 443   scoped_ptr<char[]> buf(new char[current_size]);
 444   return index_->Read(buf.get(), current_size, 0);
 445 }
 446
 447 bool BackendImpl::InitStats() {
 448   Addr address(data_->header.stats);
 449   int size = stats_.StorageSize();
 450
 451   if (!address.is_initialized()) {
 452     FileType file_type = Addr::RequiredFileType(size);
 453     DCHECK_NE(file_type, EXTERNAL);
 454     int num_blocks = Addr::RequiredBlocks(size, file_type);
 455
 456     if (!CreateBlock(file_type, num_blocks, &address))
 457       return false;
 458     return stats_.Init(NULL, 0, address);
 459   }
 460
 461   if (!address.is_block_file()) {
 462     NOTREACHED();
 463     return false;
 464   }
 465
 466   // Load the required data.
 467   size = address.num_blocks() * address.BlockSize();
 468   MappedFile* file = File(address);
 469   if (!file)
 470     return false;
 471
 472   scoped_ptr<char[]> data(new char[size]);
 473   size_t offset = address.start_block() * address.BlockSize() +
 474                   kBlockHeaderSize;
 475   if (!file->Read(data.get(), size, offset))
 476     return false;
 477
 478   if (!stats_.Init(data.get(), size, address))
 479     return false;
 480   if (cache_type_ == net::DISK_CACHE && ShouldReportAgain())
 481     stats_.InitSizeHistogram();
 482   return true;
 483 }
 484
 485 }  // namespace disk_cache