net/disk_cache/blockfile/backend_worker_v3.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/disk_cache/blockfile/backend_worker_v3.h"
   6
   7 #include "base/bind.h"
   8 #include "base/bind_helpers.h"
   9 #include "base/files/file_path.h"
  10 #include "base/files/file_util.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "base/strings/string_util.h"
  13 #include "base/strings/stringprintf.h"
  14 #include "base/time/time.h"
  15 #include "base/timer/timer.h"
  16 #include "net/base/net_errors.h"
  17 #include "net/disk_cache/blockfile/errors.h"
  18 #include "net/disk_cache/blockfile/experiments.h"
  19 #include "net/disk_cache/blockfile/file.h"
  20
  21 using base::Time;
  22 using base::TimeDelta;
  23 using base::TimeTicks;
  24
  25 namespace {
  26
  27 #if defined(V3_NOT_JUST_YET_READY)
  28
  29 const char kIndexName[] = "index";
  30
  31 // Seems like ~240 MB correspond to less than 50k entries for 99% of the people.
  32 // Note that the actual target is to keep the index table load factor under 55%
  33 // for most users.
  34 const int k64kEntriesStore = 240 * 1000 * 1000;
  35 const int kBaseTableLen = 64 * 1024;
  36 const int kDefaultCacheSize = 80 * 1024 * 1024;
  37
  38 // Avoid trimming the cache for the first 5 minutes (10 timer ticks).
  39 const int kTrimDelay = 10;
  40
  41 int DesiredIndexTableLen(int32 storage_size) {
  42   if (storage_size <= k64kEntriesStore)
  43     return kBaseTableLen;
  44   if (storage_size <= k64kEntriesStore * 2)
  45     return kBaseTableLen * 2;
  46   if (storage_size <= k64kEntriesStore * 4)
  47     return kBaseTableLen * 4;
  48   if (storage_size <= k64kEntriesStore * 8)
  49     return kBaseTableLen * 8;
  50
  51   // The biggest storage_size for int32 requires a 4 MB table.
  52   return kBaseTableLen * 16;
  53 }
  54
  55 int MaxStorageSizeForTable(int table_len) {
  56   return table_len * (k64kEntriesStore / kBaseTableLen);
  57 }
  58
  59 size_t GetIndexSize(int table_len) {
  60   size_t table_size = sizeof(disk_cache::CacheAddr) * table_len;
  61   return sizeof(disk_cache::IndexHeader) + table_size;
  62 }
  63
  64 // ------------------------------------------------------------------------
  65
  66 // Sets group for the current experiment. Returns false if the files should be
  67 // discarded.
  68 bool InitExperiment(disk_cache::IndexHeader* header, bool cache_created) {
  69   if (header->experiment == disk_cache::EXPERIMENT_OLD_FILE1 ||
  70       header->experiment == disk_cache::EXPERIMENT_OLD_FILE2) {
  71     // Discard current cache.
  72     return false;
  73   }
  74
  75   if (base::FieldTrialList::FindFullName("SimpleCacheTrial") ==
  76           "ExperimentControl") {
  77     if (cache_created) {
  78       header->experiment = disk_cache::EXPERIMENT_SIMPLE_CONTROL;
  79       return true;
  80     } else if (header->experiment != disk_cache::EXPERIMENT_SIMPLE_CONTROL) {
  81       return false;
  82     }
  83   }
  84
  85   header->experiment = disk_cache::NO_EXPERIMENT;
  86   return true;
  87 }
  88 #endif  // defined(V3_NOT_JUST_YET_READY).
  89
  90 }  // namespace
  91
  92 // ------------------------------------------------------------------------
  93
  94 namespace disk_cache {
  95
  96 BackendImplV3::Worker::Worker(
  97     const base::FilePath& path,
  98     const scoped_refptr<base::SingleThreadTaskRunner>& main_thread)
  99     : path_(path), block_files_(path), init_(false) {
 100 }
 101
 102 #if defined(V3_NOT_JUST_YET_READY)
 103
 104 int BackendImpl::SyncInit() {
 105 #if defined(NET_BUILD_STRESS_CACHE)
 106   // Start evictions right away.
 107   up_ticks_ = kTrimDelay * 2;
 108 #endif
 109   DCHECK(!init_);
 110   if (init_)
 111     return net::ERR_FAILED;
 112
 113   bool create_files = false;
 114   if (!InitBackingStore(&create_files)) {
 115     ReportError(ERR_STORAGE_ERROR);
 116     return net::ERR_FAILED;
 117   }
 118
 119   num_refs_ = num_pending_io_ = max_refs_ = 0;
 120   entry_count_ = byte_count_ = 0;
 121
 122   if (!restarted_) {
 123     buffer_bytes_ = 0;
 124     trace_object_ = TraceObject::GetTraceObject();
 125     // Create a recurrent timer of 30 secs.
 126     int timer_delay = unit_test_ ? 1000 : 30000;
 127     timer_.reset(new base::RepeatingTimer<BackendImpl>());
 128     timer_->Start(FROM_HERE, TimeDelta::FromMilliseconds(timer_delay), this,
 129                   &BackendImpl::OnStatsTimer);
 130   }
 131
 132   init_ = true;
 133   Trace("Init");
 134
 135   if (data_->header.experiment != NO_EXPERIMENT &&
 136       cache_type_ != net::DISK_CACHE) {
 137     // No experiment for other caches.
 138     return net::ERR_FAILED;
 139   }
 140
 141   if (!(user_flags_ & kNoRandom)) {
 142     // The unit test controls directly what to test.
 143     new_eviction_ = (cache_type_ == net::DISK_CACHE);
 144   }
 145
 146   if (!CheckIndex()) {
 147     ReportError(ERR_INIT_FAILED);
 148     return net::ERR_FAILED;
 149   }
 150
 151   if (!restarted_ && (create_files || !data_->header.num_entries))
 152     ReportError(ERR_CACHE_CREATED);
 153
 154   if (!(user_flags_ & kNoRandom) && cache_type_ == net::DISK_CACHE &&
 155       !InitExperiment(&data_->header, create_files)) {
 156     return net::ERR_FAILED;
 157   }
 158
 159   // We don't care if the value overflows. The only thing we care about is that
 160   // the id cannot be zero, because that value is used as "not dirty".
 161   // Increasing the value once per second gives us many years before we start
 162   // having collisions.
 163   data_->header.this_id++;
 164   if (!data_->header.this_id)
 165     data_->header.this_id++;
 166
 167   bool previous_crash = (data_->header.crash != 0);
 168   data_->header.crash = 1;
 169
 170   if (!block_files_.Init(create_files))
 171     return net::ERR_FAILED;
 172
 173   // We want to minimize the changes to cache for an AppCache.
 174   if (cache_type() == net::APP_CACHE) {
 175     DCHECK(!new_eviction_);
 176     read_only_ = true;
 177   } else if (cache_type() == net::SHADER_CACHE) {
 178     DCHECK(!new_eviction_);
 179   }
 180
 181   eviction_.Init(this);
 182
 183   // stats_ and rankings_ may end up calling back to us so we better be enabled.
 184   disabled_ = false;
 185   if (!InitStats())
 186     return net::ERR_FAILED;
 187
 188   disabled_ = !rankings_.Init(this, new_eviction_);
 189
 190 #if defined(STRESS_CACHE_EXTENDED_VALIDATION)
 191   trace_object_->EnableTracing(false);
 192   int sc = SelfCheck();
 193   if (sc < 0 && sc != ERR_NUM_ENTRIES_MISMATCH)
 194     NOTREACHED();
 195   trace_object_->EnableTracing(true);
 196 #endif
 197
 198   if (previous_crash) {
 199     ReportError(ERR_PREVIOUS_CRASH);
 200   } else if (!restarted_) {
 201     ReportError(ERR_NO_ERROR);
 202   }
 203
 204   FlushIndex();
 205
 206   return disabled_ ? net::ERR_FAILED : net::OK;
 207 }
 208
 209 void BackendImpl::PrepareForRestart() {
 210   // Reset the mask_ if it was not given by the user.
 211   if (!(user_flags_ & kMask))
 212     mask_ = 0;
 213
 214   if (!(user_flags_ & kNewEviction))
 215     new_eviction_ = false;
 216
 217   disabled_ = true;
 218   data_->header.crash = 0;
 219   index_->Flush();
 220   index_ = NULL;
 221   data_ = NULL;
 222   block_files_.CloseFiles();
 223   rankings_.Reset();
 224   init_ = false;
 225   restarted_ = true;
 226 }
 227
 228 BackendImpl::~BackendImpl() {
 229   if (user_flags_ & kNoRandom) {
 230     // This is a unit test, so we want to be strict about not leaking entries
 231     // and completing all the work.
 232     background_queue_.WaitForPendingIO();
 233   } else {
 234     // This is most likely not a test, so we want to do as little work as
 235     // possible at this time, at the price of leaving dirty entries behind.
 236     background_queue_.DropPendingIO();
 237   }
 238
 239   if (background_queue_.BackgroundIsCurrentThread()) {
 240     // Unit tests may use the same thread for everything.
 241     CleanupCache();
 242   } else {
 243     background_queue_.background_thread()->PostTask(
 244         FROM_HERE, base::Bind(&FinalCleanupCallback, base::Unretained(this)));
 245     // http://crbug.com/74623
 246     base::ThreadRestrictions::ScopedAllowWait allow_wait;
 247     done_.Wait();
 248   }
 249 }
 250
 251 void BackendImpl::CleanupCache() {
 252   Trace("Backend Cleanup");
 253   eviction_.Stop();
 254   timer_.reset();
 255
 256   if (init_) {
 257     StoreStats();
 258     if (data_)
 259       data_->header.crash = 0;
 260
 261     if (user_flags_ & kNoRandom) {
 262       // This is a net_unittest, verify that we are not 'leaking' entries.
 263       File::WaitForPendingIO(&num_pending_io_);
 264       DCHECK(!num_refs_);
 265     } else {
 266       File::DropPendingIO();
 267     }
 268   }
 269   block_files_.CloseFiles();
 270   FlushIndex();
 271   index_ = NULL;
 272   ptr_factory_.InvalidateWeakPtrs();
 273   done_.Signal();
 274 }
 275
 276 base::FilePath BackendImpl::GetFileName(Addr address) const {
 277   if (!address.is_separate_file() || !address.is_initialized()) {
 278     NOTREACHED();
 279     return base::FilePath();
 280   }
 281
 282   std::string tmp = base::StringPrintf("f_%06x", address.FileNumber());
 283   return path_.AppendASCII(tmp);
 284 }
 285
 286 // We just created a new file so we're going to write the header and set the
 287 // file length to include the hash table (zero filled).
 288 bool BackendImpl::CreateBackingStore(disk_cache::File* file) {
 289   AdjustMaxCacheSize(0);
 290
 291   IndexHeader header;
 292   header.table_len = DesiredIndexTableLen(max_size_);
 293
 294   // We need file version 2.1 for the new eviction algorithm.
 295   if (new_eviction_)
 296     header.version = 0x20001;
 297
 298   header.create_time = Time::Now().ToInternalValue();
 299
 300   if (!file->Write(&header, sizeof(header), 0))
 301     return false;
 302
 303   return file->SetLength(GetIndexSize(header.table_len));
 304 }
 305
 306 bool BackendImpl::InitBackingStore(bool* file_created) {
 307   if (!base::CreateDirectory(path_))
 308     return false;
 309
 310   base::FilePath index_name = path_.AppendASCII(kIndexName);
 311
 312   int flags = base::PLATFORM_FILE_READ |
 313               base::PLATFORM_FILE_WRITE |
 314               base::PLATFORM_FILE_OPEN_ALWAYS |
 315               base::PLATFORM_FILE_EXCLUSIVE_WRITE;
 316   scoped_refptr<disk_cache::File> file(new disk_cache::File(
 317       base::CreatePlatformFile(index_name, flags, file_created, NULL)));
 318
 319   if (!file->IsValid())
 320     return false;
 321
 322   bool ret = true;
 323   if (*file_created)
 324     ret = CreateBackingStore(file.get());
 325
 326   file = NULL;
 327   if (!ret)
 328     return false;
 329
 330   index_ = new MappedFile();
 331   data_ = reinterpret_cast<Index*>(index_->Init(index_name, 0));
 332   if (!data_) {
 333     LOG(ERROR) << "Unable to map Index file";
 334     return false;
 335   }
 336
 337   if (index_->GetLength() < sizeof(Index)) {
 338     // We verify this again on CheckIndex() but it's easier to make sure now
 339     // that the header is there.
 340     LOG(ERROR) << "Corrupt Index file";
 341     return false;
 342   }
 343
 344   return true;
 345 }
 346
 347 void BackendImpl::ReportError(int error) {
 348   STRESS_DCHECK(!error || error == ERR_PREVIOUS_CRASH ||
 349                 error == ERR_CACHE_CREATED);
 350
 351   // We transmit positive numbers, instead of direct error codes.
 352   DCHECK_LE(error, 0);
 353   CACHE_UMA(CACHE_ERROR, "Error", 0, error * -1);
 354 }
 355
 356
 357 bool BackendImpl::CheckIndex() {
 358   DCHECK(data_);
 359
 360   size_t current_size = index_->GetLength();
 361   if (current_size < sizeof(Index)) {
 362     LOG(ERROR) << "Corrupt Index file";
 363     return false;
 364   }
 365
 366   if (new_eviction_) {
 367     // We support versions 2.0 and 2.1, upgrading 2.0 to 2.1.
 368     if (kIndexMagic != data_->header.magic ||
 369         kCurrentVersion >> 16 != data_->header.version >> 16) {
 370       LOG(ERROR) << "Invalid file version or magic";
 371       return false;
 372     }
 373     if (kCurrentVersion == data_->header.version) {
 374       // We need file version 2.1 for the new eviction algorithm.
 375       UpgradeTo2_1();
 376     }
 377   } else {
 378     if (kIndexMagic != data_->header.magic ||
 379         kCurrentVersion != data_->header.version) {
 380       LOG(ERROR) << "Invalid file version or magic";
 381       return false;
 382     }
 383   }
 384
 385   if (!data_->header.table_len) {
 386     LOG(ERROR) << "Invalid table size";
 387     return false;
 388   }
 389
 390   if (current_size < GetIndexSize(data_->header.table_len) ||
 391       data_->header.table_len & (kBaseTableLen - 1)) {
 392     LOG(ERROR) << "Corrupt Index file";
 393     return false;
 394   }
 395
 396   AdjustMaxCacheSize(data_->header.table_len);
 397
 398 #if !defined(NET_BUILD_STRESS_CACHE)
 399   if (data_->header.num_bytes < 0 ||
 400       (max_size_ < kint32max - kDefaultCacheSize &&
 401        data_->header.num_bytes > max_size_ + kDefaultCacheSize)) {
 402     LOG(ERROR) << "Invalid cache (current) size";
 403     return false;
 404   }
 405 #endif
 406
 407   if (data_->header.num_entries < 0) {
 408     LOG(ERROR) << "Invalid number of entries";
 409     return false;
 410   }
 411
 412   if (!mask_)
 413     mask_ = data_->header.table_len - 1;
 414
 415   // Load the table into memory with a single read.
 416   scoped_ptr<char[]> buf(new char[current_size]);
 417   return index_->Read(buf.get(), current_size, 0);
 418 }
 419
 420 bool BackendImpl::InitStats() {
 421   Addr address(data_->header.stats);
 422   int size = stats_.StorageSize();
 423
 424   if (!address.is_initialized()) {
 425     FileType file_type = Addr::RequiredFileType(size);
 426     DCHECK_NE(file_type, EXTERNAL);
 427     int num_blocks = Addr::RequiredBlocks(size, file_type);
 428
 429     if (!CreateBlock(file_type, num_blocks, &address))
 430       return false;
 431     return stats_.Init(NULL, 0, address);
 432   }
 433
 434   if (!address.is_block_file()) {
 435     NOTREACHED();
 436     return false;
 437   }
 438
 439   // Load the required data.
 440   size = address.num_blocks() * address.BlockSize();
 441   MappedFile* file = File(address);
 442   if (!file)
 443     return false;
 444
 445   scoped_ptr<char[]> data(new char[size]);
 446   size_t offset = address.start_block() * address.BlockSize() +
 447                   kBlockHeaderSize;
 448   if (!file->Read(data.get(), size, offset))
 449     return false;
 450
 451   if (!stats_.Init(data.get(), size, address))
 452     return false;
 453   if (cache_type_ == net::DISK_CACHE && ShouldReportAgain())
 454     stats_.InitSizeHistogram();
 455   return true;
 456 }
 457
 458 #endif  // defined(V3_NOT_JUST_YET_READY).
 459
 460 int BackendImplV3::Worker::Init(const CompletionCallback& callback) {
 461   return net::ERR_FAILED;
 462 }
 463
 464 BackendImplV3::Worker::~Worker() {
 465 }
 466
 467 }  // namespace disk_cache