cpp/src/IceGrid/FileCache.cpp

   1 // **********************************************************************
   2 //
   3 // Copyright (c) 2003-2011 ZeroC, Inc. All rights reserved.
   4 //
   5 // This copy of Ice is licensed to you under the terms described in the
   6 // ICE_LICENSE file included in this distribution.
   7 //
   8 // **********************************************************************
   9
  10 #include <Ice/Communicator.h>
  11 #include <Ice/Properties.h>
  12
  13 #include <IceUtil/FileUtil.h>
  14
  15 #include <IceGrid/FileCache.h>
  16 #include <IceGrid/Exception.h>
  17
  18 #include <deque>
  19
  20 using namespace std;
  21 using namespace IceGrid;
  22
  23 FileCache::FileCache(const Ice::CommunicatorPtr& com) :
  24     _messageSizeMax(com->getProperties()->getPropertyAsIntWithDefault("Ice.MessageSizeMax", 1024) * 1024 - 256)
  25 {
  26 }
  27
  28 Ice::Long
  29 FileCache::getOffsetFromEnd(const string& file, int originalCount)
  30 {
  31     IceUtilInternal::ifstream is(file); // file is a UTF-8 string
  32     if(is.fail())
  33     {
  34         throw FileNotAvailableException("failed to open file `" + file + "'");
  35     }
  36
  37     if(originalCount < 0)
  38     {
  39         return 0;
  40     }
  41
  42     is.seekg(0, ios::end);
  43     streampos endOfFile = is.tellg();
  44     if(originalCount == 0)
  45     {
  46         return endOfFile;
  47     }
  48
  49     streamoff blockSize = 16 * 1024; // Start reading a block of 16K from the end of the file.
  50     streampos lastBlockOffset = endOfFile;
  51     int totalCount = 0;
  52     string line;
  53     deque<pair<streampos, string> > lines;
  54     do
  55     {
  56         lines.clear();
  57
  58         //
  59         // Move the current position of the stream to the new block to
  60         // read.
  61         //
  62         is.clear();
  63         if(lastBlockOffset - blockSize > streamoff(0))
  64         {
  65             is.seekg(lastBlockOffset - blockSize);
  66             getline(is, line); // Ignore the first line as it's most likely not complete.
  67         }
  68         else
  69         {
  70             is.seekg(0, ios::beg); // We've reach the begining of the file.
  71         }
  72
  73         //
  74         // Read the block and count the number of lines in the block
  75         // If we found the "first last N lines", we start throwing out
  76         // the lines read at the begining of the file.
  77         //
  78         int count = originalCount - totalCount; // Number of lines left to find.
  79         while(is.good() && is.tellg() <= streamoff(lastBlockOffset))
  80         {
  81             streampos beg = is.tellg();
  82             getline(is, line);
  83             if(is.eof() && line.empty()) // Don't count the last line if it's empty.
  84             {
  85                 continue;
  86             }
  87
  88             lines.push_back(make_pair(beg, line));
  89             ++totalCount;
  90             if(lines.size() == static_cast<unsigned int>(count + 1))
  91             {
  92                 --totalCount;
  93                 lines.pop_front();
  94             }
  95         }
  96
  97         if(lastBlockOffset - blockSize < streamoff(0))
  98         {
  99             break; // We're done if the block started at the begining of the file.
 100         }
 101         else if(totalCount < originalCount)
 102         {
 103             //
 104             // Otherwise, it we still didn't find the required number of lines,
 105             // read another block of text before this block.
 106             //
 107             lastBlockOffset -= blockSize; // Position of the block we just read.
 108             blockSize *= 2; // Read a bigger block.
 109         }
 110     }
 111     while(totalCount < originalCount && !is.bad());
 112
 113     if(is.bad())
 114     {
 115         throw FileNotAvailableException("unrecoverable error occured while reading file `" + file + "'");
 116     }
 117
 118     if(lines.empty())
 119     {
 120         return 0;
 121     }
 122     else
 123     {
 124         return lines[0].first;
 125     }
 126 }
 127
 128 bool
 129 FileCache::read(const string& file, Ice::Long offset, int size, Ice::Long& newOffset, Ice::StringSeq& lines)
 130 {
 131     assert(size > 0);
 132
 133     if(size > _messageSizeMax)
 134     {
 135         size = _messageSizeMax;
 136     }
 137
 138     if(size <= 5)
 139     {
 140         throw FileNotAvailableException("maximum bytes per read request is too low");
 141     }
 142
 143     IceUtilInternal::ifstream is(file); // file is a UTF-8 string
 144     if(is.fail())
 145     {
 146         throw FileNotAvailableException("failed to open file `" + file + "'");
 147     }
 148
 149     //
 150     // Check if the requested offset is past the end of the file, if
 151     // that's the case return an empty sequence of lines and indicate
 152     // the EOF.
 153     //
 154     is.seekg(0, ios::end);
 155     if(offset >= is.tellg())
 156     {
 157         newOffset = is.tellg();
 158         lines = Ice::StringSeq();
 159         return true;
 160     }
 161
 162     //
 163     // Read lines from the file until we read enough or reached EOF.
 164     //
 165     newOffset = offset;
 166     lines = Ice::StringSeq();
 167 #if defined(_MSC_VER) && (_MSC_VER < 1300)
 168     is.seekg(static_cast<int>(offset));
 169 #else
 170     is.seekg(static_cast<streamoff>(offset), ios::beg);
 171 #endif
 172     int totalSize = 0;
 173     string line;
 174
 175     for(int i = 0; is.good(); ++i)
 176     {
 177         getline(is, line);
 178
 179         int lineSize = static_cast<int>(line.size()) + 5; // 5 bytes for the encoding of the string size (worst case)
 180         if(lineSize + totalSize > size)
 181         {
 182             if(totalSize + 5 < size)
 183             {
 184                 // There's some room left for a part of the string, return a partial string
 185                 line = line.substr(0, size - totalSize - 5);
 186                 lines.push_back(line);
 187                 newOffset += line.size();
 188             }
 189             else
 190             {
 191                 lines.push_back("");
 192             }
 193             return false; // We didn't reach the end of file, we've just reached the size limit!
 194         }
 195
 196         totalSize += lineSize;
 197         lines.push_back(line);
 198 #if defined(_MSC_VER) && (_MSC_VER < 1300)
 199         if(is.eof())
 200         {
 201             newOffset += line.size();
 202         }
 203         else
 204 #else
 205         if(!is.fail())
 206 #endif
 207         {
 208             newOffset = is.tellg();
 209         }
 210     }
 211
 212     if(is.bad())
 213     {
 214         throw FileNotAvailableException("unrecoverable error occured while reading file `" + file + "'");
 215     }
 216
 217     return is.eof();
 218 }
 219