external_libraries/yaml-cpp-0.3.0/src/scanner.cpp

   1 #include "scanner.h"
   2 #include "token.h"
   3 #include "yaml-cpp/exceptions.h"
   4 #include "exp.h"
   5 #include <cassert>
   6 #include <memory>
   7
   8 namespace YAML
   9 {
  10         Scanner::Scanner(std::istream& in)
  11                 : INPUT(in), m_startedStream(false), m_endedStream(false), m_simpleKeyAllowed(false), m_canBeJSONFlow(false)
  12         {
  13         }
  14
  15         Scanner::~Scanner()
  16         {
  17         }
  18
  19         // empty
  20         // . Returns true if there are no more tokens to be read
  21         bool Scanner::empty()
  22         {
  23                 EnsureTokensInQueue();
  24                 return m_tokens.empty();
  25         }
  26
  27         // pop
  28         // . Simply removes the next token on the queue.
  29         void Scanner::pop()
  30         {
  31                 EnsureTokensInQueue();
  32                 if(!m_tokens.empty())
  33                         m_tokens.pop();
  34         }
  35
  36         // peek
  37         // . Returns (but does not remove) the next token on the queue.
  38         Token& Scanner::peek()
  39         {
  40                 EnsureTokensInQueue();
  41                 assert(!m_tokens.empty());  // should we be asserting here? I mean, we really just be checking
  42                                             // if it's empty before peeking.
  43
  44 #if 0
  45                 static Token *pLast = 0;
  46                 if(pLast != &m_tokens.front())
  47                         std::cerr << "peek: " << m_tokens.front() << "\n";
  48                 pLast = &m_tokens.front();
  49 #endif
  50
  51                 return m_tokens.front();
  52         }
  53
  54         // EnsureTokensInQueue
  55         // . Scan until there's a valid token at the front of the queue,
  56         //   or we're sure the queue is empty.
  57         void Scanner::EnsureTokensInQueue()
  58         {
  59                 while(1) {
  60                         if(!m_tokens.empty()) {
  61                                 Token& token = m_tokens.front();
  62
  63                                 // if this guy's valid, then we're done
  64                                 if(token.status == Token::VALID)
  65                                         return;
  66
  67                                 // here's where we clean up the impossible tokens
  68                                 if(token.status == Token::INVALID) {
  69                                         m_tokens.pop();
  70                                         continue;
  71                                 }
  72
  73                                 // note: what's left are the unverified tokens
  74                         }
  75
  76                         // no token? maybe we've actually finished
  77                         if(m_endedStream)
  78                                 return;
  79
  80                         // no? then scan...
  81                         ScanNextToken();
  82                 }
  83         }
  84
  85         // ScanNextToken
  86         // . The main scanning function; here we branch out and
  87         //   scan whatever the next token should be.
  88         void Scanner::ScanNextToken()
  89         {
  90                 if(m_endedStream)
  91                         return;
  92
  93                 if(!m_startedStream)
  94                         return StartStream();
  95
  96                 // get rid of whitespace, etc. (in between tokens it should be irrelevent)
  97                 ScanToNextToken();
  98
  99                 // maybe need to end some blocks
 100                 PopIndentToHere();
 101
 102                 // *****
 103                 // And now branch based on the next few characters!
 104                 // *****
 105
 106                 // end of stream
 107                 if(!INPUT)
 108                         return EndStream();
 109
 110                 if(INPUT.column() == 0 && INPUT.peek() == Keys::Directive)
 111                         return ScanDirective();
 112
 113                 // document token
 114                 if(INPUT.column() == 0 && Exp::DocStart().Matches(INPUT))
 115                         return ScanDocStart();
 116
 117                 if(INPUT.column() == 0 && Exp::DocEnd().Matches(INPUT))
 118                         return ScanDocEnd();
 119
 120                 // flow start/end/entry
 121                 if(INPUT.peek() == Keys::FlowSeqStart || INPUT.peek() == Keys::FlowMapStart)
 122                         return ScanFlowStart();
 123
 124                 if(INPUT.peek() == Keys::FlowSeqEnd || INPUT.peek() == Keys::FlowMapEnd)
 125                         return ScanFlowEnd();
 126
 127                 if(INPUT.peek() == Keys::FlowEntry)
 128                         return ScanFlowEntry();
 129
 130                 // block/map stuff
 131                 if(Exp::BlockEntry().Matches(INPUT))
 132                         return ScanBlockEntry();
 133
 134                 if((InBlockContext() ? Exp::Key() : Exp::KeyInFlow()).Matches(INPUT))
 135                         return ScanKey();
 136
 137                 if(GetValueRegex().Matches(INPUT))
 138                         return ScanValue();
 139
 140                 // alias/anchor
 141                 if(INPUT.peek() == Keys::Alias || INPUT.peek() == Keys::Anchor)
 142                         return ScanAnchorOrAlias();
 143
 144                 // tag
 145                 if(INPUT.peek() == Keys::Tag)
 146                         return ScanTag();
 147
 148                 // special scalars
 149                 if(InBlockContext() && (INPUT.peek() == Keys::LiteralScalar || INPUT.peek() == Keys::FoldedScalar))
 150                         return ScanBlockScalar();
 151
 152                 if(INPUT.peek() == '\'' || INPUT.peek() == '\"')
 153                         return ScanQuotedScalar();
 154
 155                 // plain scalars
 156                 if((InBlockContext() ? Exp::PlainScalar() : Exp::PlainScalarInFlow()).Matches(INPUT))
 157                         return ScanPlainScalar();
 158
 159                 // don't know what it is!
 160                 throw ParserException(INPUT.mark(), ErrorMsg::UNKNOWN_TOKEN);
 161         }
 162
 163         // ScanToNextToken
 164         // . Eats input until we reach the next token-like thing.
 165         void Scanner::ScanToNextToken()
 166         {
 167                 while(1) {
 168                         // first eat whitespace
 169                         while(INPUT && IsWhitespaceToBeEaten(INPUT.peek())) {
 170                                 if(InBlockContext() && Exp::Tab().Matches(INPUT))
 171                                         m_simpleKeyAllowed = false;
 172                                 INPUT.eat(1);
 173                         }
 174
 175                         // then eat a comment
 176                         if(Exp::Comment().Matches(INPUT)) {
 177                                 // eat until line break
 178                                 while(INPUT && !Exp::Break().Matches(INPUT))
 179                                         INPUT.eat(1);
 180                         }
 181
 182                         // if it's NOT a line break, then we're done!
 183                         if(!Exp::Break().Matches(INPUT))
 184                                 break;
 185
 186                         // otherwise, let's eat the line break and keep going
 187                         int n = Exp::Break().Match(INPUT);
 188                         INPUT.eat(n);
 189
 190                         // oh yeah, and let's get rid of that simple key
 191                         InvalidateSimpleKey();
 192
 193                         // new line - we may be able to accept a simple key now
 194                         if(InBlockContext())
 195                                 m_simpleKeyAllowed = true;
 196         }
 197         }
 198
 199         ///////////////////////////////////////////////////////////////////////
 200         // Misc. helpers
 201
 202         // IsWhitespaceToBeEaten
 203         // . We can eat whitespace if it's a space or tab
 204         // . Note: originally tabs in block context couldn't be eaten
 205         //         "where a simple key could be allowed
 206         //         (i.e., not at the beginning of a line, or following '-', '?', or ':')"
 207         //   I think this is wrong, since tabs can be non-content whitespace; it's just
 208         //   that they can't contribute to indentation, so once you've seen a tab in a
 209         //   line, you can't start a simple key
 210         bool Scanner::IsWhitespaceToBeEaten(char ch)
 211         {
 212                 if(ch == ' ')
 213                         return true;
 214
 215                 if(ch == '\t')
 216                         return true;
 217
 218                 return false;
 219         }
 220
 221         // GetValueRegex
 222         // . Get the appropriate regex to check if it's a value token
 223         const RegEx& Scanner::GetValueRegex() const
 224         {
 225                 if(InBlockContext())
 226                         return Exp::Value();
 227
 228                 return m_canBeJSONFlow ? Exp::ValueInJSONFlow() : Exp::ValueInFlow();
 229         }
 230
 231         // StartStream
 232         // . Set the initial conditions for starting a stream.
 233         void Scanner::StartStream()
 234         {
 235                 m_startedStream = true;
 236                 m_simpleKeyAllowed = true;
 237                 std::auto_ptr<IndentMarker> pIndent(new IndentMarker(-1, IndentMarker::NONE));
 238                 m_indentRefs.push_back(pIndent);
 239                 m_indents.push(&m_indentRefs.back());
 240         }
 241
 242         // EndStream
 243         // . Close out the stream, finish up, etc.
 244         void Scanner::EndStream()
 245         {
 246                 // force newline
 247                 if(INPUT.column() > 0)
 248                         INPUT.ResetColumn();
 249
 250                 PopAllIndents();
 251                 PopAllSimpleKeys();
 252
 253                 m_simpleKeyAllowed = false;
 254                 m_endedStream = true;
 255         }
 256
 257         Token *Scanner::PushToken(Token::TYPE type)
 258         {
 259                 m_tokens.push(Token(type, INPUT.mark()));
 260                 return &m_tokens.back();
 261         }
 262
 263         Token::TYPE Scanner::GetStartTokenFor(IndentMarker::INDENT_TYPE type) const
 264         {
 265                 switch(type) {
 266                         case IndentMarker::SEQ: return Token::BLOCK_SEQ_START;
 267                         case IndentMarker::MAP: return Token::BLOCK_MAP_START;
 268                         case IndentMarker::NONE: assert(false); break;
 269                 }
 270                 assert(false);
 271                 throw std::runtime_error("yaml-cpp: internal error, invalid indent type");
 272         }
 273
 274         // PushIndentTo
 275         // . Pushes an indentation onto the stack, and enqueues the
 276         //   proper token (sequence start or mapping start).
 277         // . Returns the indent marker it generates (if any).
 278         Scanner::IndentMarker *Scanner::PushIndentTo(int column, IndentMarker::INDENT_TYPE type)
 279         {
 280                 // are we in flow?
 281                 if(InFlowContext())
 282                         return 0;
 283
 284                 std::auto_ptr<IndentMarker> pIndent(new IndentMarker(column, type));
 285                 IndentMarker& indent = *pIndent;
 286                 const IndentMarker& lastIndent = *m_indents.top();
 287
 288                 // is this actually an indentation?
 289                 if(indent.column < lastIndent.column)
 290                         return 0;
 291                 if(indent.column == lastIndent.column && !(indent.type == IndentMarker::SEQ && lastIndent.type == IndentMarker::MAP))
 292                         return 0;
 293
 294                 // push a start token
 295                 indent.pStartToken = PushToken(GetStartTokenFor(type));
 296
 297                 // and then the indent
 298                 m_indents.push(&indent);
 299                 m_indentRefs.push_back(pIndent);
 300                 return &m_indentRefs.back();
 301         }
 302
 303         // PopIndentToHere
 304         // . Pops indentations off the stack until we reach the current indentation level,
 305         //   and enqueues the proper token each time.
 306         // . Then pops all invalid indentations off.
 307         void Scanner::PopIndentToHere()
 308         {
 309                 // are we in flow?
 310                 if(InFlowContext())
 311                         return;
 312
 313                 // now pop away
 314                 while(!m_indents.empty()) {
 315                         const IndentMarker& indent = *m_indents.top();
 316                         if(indent.column < INPUT.column())
 317                                 break;
 318                         if(indent.column == INPUT.column() && !(indent.type == IndentMarker::SEQ && !Exp::BlockEntry().Matches(INPUT)))
 319                                 break;
 320
 321                         PopIndent();
 322                 }
 323
 324                 while(!m_indents.empty() && m_indents.top()->status == IndentMarker::INVALID)
 325                         PopIndent();
 326         }
 327
 328         // PopAllIndents
 329         // . Pops all indentations (except for the base empty one) off the stack,
 330         //   and enqueues the proper token each time.
 331         void Scanner::PopAllIndents()
 332         {
 333                 // are we in flow?
 334                 if(InFlowContext())
 335                         return;
 336
 337                 // now pop away
 338                 while(!m_indents.empty()) {
 339                         const IndentMarker& indent = *m_indents.top();
 340                         if(indent.type == IndentMarker::NONE)
 341                                 break;
 342
 343                         PopIndent();
 344                 }
 345         }
 346
 347         // PopIndent
 348         // . Pops a single indent, pushing the proper token
 349         void Scanner::PopIndent()
 350         {
 351                 const IndentMarker& indent = *m_indents.top();
 352                 m_indents.pop();
 353
 354                 if(indent.status != IndentMarker::VALID) {
 355                         InvalidateSimpleKey();
 356                         return;
 357                 }
 358
 359                 if(indent.type == IndentMarker::SEQ)
 360                         m_tokens.push(Token(Token::BLOCK_SEQ_END, INPUT.mark()));
 361                 else if(indent.type == IndentMarker::MAP)
 362                         m_tokens.push(Token(Token::BLOCK_MAP_END, INPUT.mark()));
 363         }
 364
 365         // GetTopIndent
 366         int Scanner::GetTopIndent() const
 367         {
 368                 if(m_indents.empty())
 369                         return 0;
 370                 return m_indents.top()->column;
 371         }
 372
 373         // ThrowParserException
 374         // . Throws a ParserException with the current token location
 375         //   (if available).
 376         // . Does not parse any more tokens.
 377         void Scanner::ThrowParserException(const std::string& msg) const
 378         {
 379                 Mark mark = Mark::null();
 380                 if(!m_tokens.empty()) {
 381                         const Token& token = m_tokens.front();
 382                         mark = token.mark;
 383                 }
 384                 throw ParserException(mark, msg);
 385         }
 386 }
 387