chrome/browser/safe_browsing/prefix_set.h

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // A read-only set implementation for |SBPrefix| items.  Prefixes are
   6 // sorted and stored as 16-bit deltas from the previous prefix.  An
   7 // index structure provides quick random access, and also handles
   8 // cases where 16 bits cannot encode a delta.
   9 //
  10 // For example, the sequence {20, 25, 41, 65432, 150000, 160000} would
  11 // be stored as:
  12 //  A pair {20, 0} in |index_|.
  13 //  5, 16, 65391 in |deltas_|.
  14 //  A pair {150000, 3} in |index_|.
  15 //  10000 in |deltas_|.
  16 // |index_.size()| will be 2, |deltas_.size()| will be 4.
  17 //
  18 // This structure is intended for storage of sparse uniform sets of
  19 // prefixes of a certain size.  As of this writing, my safe-browsing
  20 // database contains:
  21 //   653132 add prefixes
  22 //   6446 are duplicates (from different chunks)
  23 //   24301 w/in 2^8 of the prior prefix
  24 //   622337 w/in 2^16 of the prior prefix
  25 //   47 further than 2^16 from the prior prefix
  26 // For this input, the memory usage is approximately 2 bytes per
  27 // prefix, a bit over 1.2M.  The bloom filter used 25 bits per prefix,
  28 // a bit over 1.9M on this data.
  29 //
  30 // Experimenting with random selections of the above data, storage
  31 // size drops almost linearly as prefix count drops, until the index
  32 // overhead starts to become a problem a bit under 200k prefixes.  The
  33 // memory footprint gets worse than storing the raw prefix data around
  34 // 75k prefixes.  Fortunately, the actual memory footprint also falls.
  35 // If the prefix count increases the memory footprint should increase
  36 // approximately linearly.  The worst-case would be 2^16 items all
  37 // 2^16 apart, which would need 512k (versus 256k to store the raw
  38 // data).
  39 //
  40 // The on-disk format looks like:
  41 //         4 byte magic number
  42 //         4 byte version number
  43 //         4 byte |index_.size()|
  44 //         4 byte |deltas_.size()|
  45 //     n * 8 byte |&index_[0]..&index_[n]|
  46 //     m * 2 byte |&deltas_[0]..&deltas_[m]|
  47 //        16 byte digest
  48
  49 #ifndef CHROME_BROWSER_SAFE_BROWSING_PREFIX_SET_H_
  50 #define CHROME_BROWSER_SAFE_BROWSING_PREFIX_SET_H_
  51
  52 #include <vector>
  53
  54 #include "base/gtest_prod_util.h"
  55 #include "base/memory/scoped_ptr.h"
  56 #include "chrome/browser/safe_browsing/safe_browsing_util.h"
  57
  58 namespace base {
  59 class FilePath;
  60 }
  61
  62 namespace safe_browsing {
  63
  64 class PrefixSet {
  65  public:
  66   ~PrefixSet();
  67
  68   // |true| if |hash| is in the hashes passed to the set's builder, or if
  69   // |hash.prefix| is one of the prefixes passed to the set's builder.
  70   bool Exists(const SBFullHash& hash) const;
  71
  72   // Persist the set on disk.
  73   static scoped_ptr<PrefixSet> LoadFile(const base::FilePath& filter_name);
  74   bool WriteFile(const base::FilePath& filter_name) const;
  75
  76  private:
  77   friend class PrefixSetBuilder;
  78
  79   friend class PrefixSetTest;
  80   FRIEND_TEST_ALL_PREFIXES(PrefixSetTest, AllBig);
  81   FRIEND_TEST_ALL_PREFIXES(PrefixSetTest, EdgeCases);
  82   FRIEND_TEST_ALL_PREFIXES(PrefixSetTest, Empty);
  83   FRIEND_TEST_ALL_PREFIXES(PrefixSetTest, FullHashBuild);
  84   FRIEND_TEST_ALL_PREFIXES(PrefixSetTest, IntMinMax);
  85   FRIEND_TEST_ALL_PREFIXES(PrefixSetTest, OneElement);
  86   FRIEND_TEST_ALL_PREFIXES(PrefixSetTest, ReadWrite);
  87   FRIEND_TEST_ALL_PREFIXES(PrefixSetTest, ReadWriteSigned);
  88   FRIEND_TEST_ALL_PREFIXES(PrefixSetTest, Version3);
  89
  90   FRIEND_TEST_ALL_PREFIXES(SafeBrowsingStoreFileTest, BasicStore);
  91   FRIEND_TEST_ALL_PREFIXES(SafeBrowsingStoreFileTest, DeleteChunks);
  92   FRIEND_TEST_ALL_PREFIXES(SafeBrowsingStoreFileTest, DetectsCorruption);
  93   FRIEND_TEST_ALL_PREFIXES(SafeBrowsingStoreFileTest, Empty);
  94   FRIEND_TEST_ALL_PREFIXES(SafeBrowsingStoreFileTest, PrefixMinMax);
  95   FRIEND_TEST_ALL_PREFIXES(SafeBrowsingStoreFileTest, SubKnockout);
  96   FRIEND_TEST_ALL_PREFIXES(SafeBrowsingStoreFileTest, Version7);
  97   FRIEND_TEST_ALL_PREFIXES(SafeBrowsingStoreFileTest, Version8);
  98
  99   // Maximum number of consecutive deltas to encode before generating
 100   // a new index entry.  This helps keep the worst-case performance
 101   // for |Exists()| under control.
 102   static const size_t kMaxRun = 100;
 103
 104   // Helpers to make |index_| easier to deal with.
 105   typedef std::pair<SBPrefix,uint32> IndexPair;
 106   typedef std::vector<IndexPair> IndexVector;
 107   static bool PrefixLess(const IndexPair& a, const IndexPair& b);
 108
 109   // Helper to let |PrefixSetBuilder| add a run of data.  |index_prefix| is
 110   // added to |index_|, with the other elements added into |deltas_|.
 111   void AddRun(SBPrefix index_prefix,
 112               const uint16* run_begin, const uint16* run_end);
 113
 114   // |true| if |prefix| is one of the prefixes passed to the set's builder.
 115   // Provided for testing purposes.
 116   bool PrefixExists(SBPrefix prefix) const;
 117
 118   // Regenerate the vector of prefixes passed to the constructor into
 119   // |prefixes|.  Prefixes will be added in sorted order.  Useful for testing.
 120   void GetPrefixes(std::vector<SBPrefix>* prefixes) const;
 121
 122   // Used by |PrefixSetBuilder|.
 123   PrefixSet();
 124
 125   // Helper for |LoadFile()|.  Steals vector contents using |swap()|.
 126   PrefixSet(IndexVector* index,
 127             std::vector<uint16>* deltas,
 128             std::vector<SBFullHash>* full_hashes);
 129
 130   // Top-level index of prefix to offset in |deltas_|.  Each pair
 131   // indicates a base prefix and where the deltas from that prefix
 132   // begin in |deltas_|.  The deltas for a pair end at the next pair's
 133   // index into |deltas_|.
 134   IndexVector index_;
 135
 136   // Deltas which are added to the prefix in |index_| to generate
 137   // prefixes.  Deltas are only valid between consecutive items from
 138   // |index_|, or the end of |deltas_| for the last |index_| pair.
 139   std::vector<uint16> deltas_;
 140
 141   // Full hashes ordered by SBFullHashLess.
 142   std::vector<SBFullHash> full_hashes_;
 143
 144   DISALLOW_COPY_AND_ASSIGN(PrefixSet);
 145 };
 146
 147 // Helper to incrementally build a PrefixSet from a stream of sorted prefixes.
 148 class PrefixSetBuilder {
 149  public:
 150   PrefixSetBuilder();
 151   ~PrefixSetBuilder();
 152
 153   // Helper for unit tests and format conversion.
 154   explicit PrefixSetBuilder(const std::vector<SBPrefix>& prefixes);
 155
 156   // Add a prefix to the set.  Prefixes must arrive in ascending order.
 157   // Duplicate prefixes are dropped.
 158   void AddPrefix(SBPrefix prefix);
 159
 160   // Flush any buffered prefixes, and return the final PrefixSet instance.
 161   // |hashes| are sorted and stored in |full_hashes_|.  Any call other than the
 162   // destructor is illegal after this call.
 163   scoped_ptr<PrefixSet> GetPrefixSet(const std::vector<SBFullHash>& hashes);
 164
 165   // Helper for clients which only track prefixes.  Calls GetPrefixSet() with
 166   // empty hash vector.
 167   scoped_ptr<PrefixSet> GetPrefixSetNoHashes();
 168
 169  private:
 170   // Encode a run of deltas for |AddRun()|.  The run is broken by a too-large
 171   // delta, or kMaxRun, whichever comes first.
 172   void EmitRun();
 173
 174   // Buffers prefixes until enough are avaliable to emit a run.
 175   std::vector<SBPrefix> buffer_;
 176
 177   // The PrefixSet being built.
 178   scoped_ptr<PrefixSet> prefix_set_;
 179 };
 180
 181 }  // namespace safe_browsing
 182
 183 #endif  // CHROME_BROWSER_SAFE_BROWSING_PREFIX_SET_H_