cpp/galago/src/BinOrderedBinnedIterator.cpp

   1
   2 //
   3 // InvertedListIterator
   4 //
   5 // 6 January 2007 -- tds
   6 //
   7
   8 #include "BinOrderedBinnedIterator.hpp"
   9 #include "BinnedFeatureIndex.hpp"
  10 #include "UncompressedRead.hpp"
  11 #include "lemur/RVLCompress.hpp"
  12 #include "indri/RVLDecompressStream.hpp"
  13
  14 //
  15 // InvertedListIterator constructor
  16 //
  17
  18 BinOrderedBinnedIterator::BinOrderedBinnedIterator( const char* data, UINT64 start, UINT64 end )
  19     :
  20     invertedData(data)
  21 {
  22     _readTermHeader( start, end );
  23 }
  24
  25 //
  26 // bin
  27 //
  28
  29 int BinOrderedBinnedIterator::bin() {
  30     return _currentBinInfo().bin;
  31 }
  32
  33 //
  34 // next
  35 //
  36
  37 bool BinOrderedBinnedIterator::next() {
  38     _termBinIndex++;
  39     if (_termBinIndex == _termBins.size()) {
  40         _termBinIndex--;
  41         return false;
  42     }
  43
  44     return true;
  45 }
  46
  47 //
  48 // reset
  49 //
  50
  51 void BinOrderedBinnedIterator::reset() {
  52     _termBinIndex = -1;
  53 }
  54
  55 //
  56 // _currentBinInfo
  57 //
  58
  59 const BinOrderedBinnedIterator::BinInfo& BinOrderedBinnedIterator::_currentBinInfo() const {
  60     return _termBins[_termBinIndex];
  61 }
  62
  63 //
  64 // binData
  65 //
  66
  67 const char* BinOrderedBinnedIterator::binData() {
  68     const BinInfo& info = _currentBinInfo();
  69     return invertedData + info.dataStart;
  70 }
  71
  72 //
  73 // binLength
  74 //
  75
  76 UINT64 BinOrderedBinnedIterator::binDataLength() {
  77     const BinInfo& info = _currentBinInfo();
  78     return info.dataLength;
  79 }
  80
  81 //
  82 // skipData
  83 //
  84
  85 const char* BinOrderedBinnedIterator::skipData() {
  86     const BinInfo& info = _currentBinInfo();
  87     return invertedData + info.skipStart;
  88 }
  89
  90 //
  91 // skipDataLength
  92 //
  93
  94 UINT64 BinOrderedBinnedIterator::skipDataLength() {
  95     const BinInfo& info = _currentBinInfo();
  96     return info.skipLength;
  97 }
  98
  99 //
 100 // postingsDataLength
 101 //
 102
 103 UINT64 BinOrderedBinnedIterator::postingsDataLength() const {
 104     return _postingsLength;
 105 }
 106
 107 //
 108 // documentCount
 109 //
 110
 111 UINT64 BinOrderedBinnedIterator::documentCount() const {
 112     return _documentCount;
 113 }
 114
 115 //
 116 // _readTermHeader
 117 //
 118
 119 void BinOrderedBinnedIterator::_readTermHeader( UINT64 startPosition, UINT64 endPosition ) {
 120     const char* headerStart = invertedData + startPosition;
 121     const char* headerNext;
 122     int headerLength;
 123     int options;
 124
 125     headerNext = lemur::utility::RVLCompress::decompress_int( headerStart, headerLength );
 126     headerNext = lemur::utility::RVLCompress::decompress_int( headerNext, options );
 127     headerNext = lemur::utility::RVLCompress::decompress_longlong( headerNext, _documentCount );
 128     bool useSkips = ((options & 0x01) != 0);
 129
 130     indri::utility::RVLDecompressStream stream( headerNext, headerLength );
 131     headerLength += (headerNext - headerStart);
 132     UINT64 totalBinLength = 0;
 133     _postingsLength = 0;
 134
 135     while( !stream.done() ) {
 136         int termBin;
 137         int termSkipLength = 0;
 138         int termBinLength;
 139
 140         stream >> termBin;
 141         if( useSkips ) stream >> termSkipLength;
 142         stream >> termBinLength;
 143
 144         BinInfo info;
 145
 146         info.bin = termBin;
 147         info.binStart = startPosition + headerLength + totalBinLength;
 148         info.skipStart = info.binStart;
 149         info.skipLength = termSkipLength;
 150         info.dataStart = info.skipStart + info.skipLength;
 151         info.dataLength = termBinLength;
 152         info.binLength = info.skipLength + info.dataLength;
 153
 154         _termBins.push_back(info);
 155         totalBinLength += info.binLength;
 156         _postingsLength += info.dataLength;
 157     }
 158
 159     assert( startPosition + headerLength + totalBinLength == endPosition );
 160     _termBinIndex = -1;
 161 }
 162
 163 //
 164 // _word_block_end_offset
 165 //
 166
 167 static UINT16 _word_block_end_offset( const UINT16* wordBlockEnds, int index ) {
 168     if ( index == 0 ) {
 169         return 0;
 170     } else {
 171         return UncompressedRead::peek_u16( (const UINT8*) (&wordBlockEnds[index-1]) );
 172     }
 173 }
 174