scide: implement selectionLength for openDocument
[supercollider.git] / external_libraries / yaml-cpp-0.3.0 / src / scantoken.cpp
blob06d9cd623d6073b0cb383e583f0308ac508c1945
1 #include "scanner.h"
2 #include "token.h"
3 #include "yaml-cpp/exceptions.h"
4 #include "exp.h"
5 #include "scanscalar.h"
6 #include "scantag.h"
7 #include "tag.h"
8 #include <sstream>
10 namespace YAML
12 ///////////////////////////////////////////////////////////////////////
13 // Specialization for scanning specific tokens
15 // Directive
16 // . Note: no semantic checking is done here (that's for the parser to do)
17 void Scanner::ScanDirective()
19 std::string name;
20 std::vector <std::string> params;
22 // pop indents and simple keys
23 PopAllIndents();
24 PopAllSimpleKeys();
26 m_simpleKeyAllowed = false;
27 m_canBeJSONFlow = false;
29 // store pos and eat indicator
30 Token token(Token::DIRECTIVE, INPUT.mark());
31 INPUT.eat(1);
33 // read name
34 while(INPUT && !Exp::BlankOrBreak().Matches(INPUT))
35 token.value += INPUT.get();
37 // read parameters
38 while(1) {
39 // first get rid of whitespace
40 while(Exp::Blank().Matches(INPUT))
41 INPUT.eat(1);
43 // break on newline or comment
44 if(!INPUT || Exp::Break().Matches(INPUT) || Exp::Comment().Matches(INPUT))
45 break;
47 // now read parameter
48 std::string param;
49 while(INPUT && !Exp::BlankOrBreak().Matches(INPUT))
50 param += INPUT.get();
52 token.params.push_back(param);
55 m_tokens.push(token);
58 // DocStart
59 void Scanner::ScanDocStart()
61 PopAllIndents();
62 PopAllSimpleKeys();
63 m_simpleKeyAllowed = false;
64 m_canBeJSONFlow = false;
66 // eat
67 Mark mark = INPUT.mark();
68 INPUT.eat(3);
69 m_tokens.push(Token(Token::DOC_START, mark));
72 // DocEnd
73 void Scanner::ScanDocEnd()
75 PopAllIndents();
76 PopAllSimpleKeys();
77 m_simpleKeyAllowed = false;
78 m_canBeJSONFlow = false;
80 // eat
81 Mark mark = INPUT.mark();
82 INPUT.eat(3);
83 m_tokens.push(Token(Token::DOC_END, mark));
86 // FlowStart
87 void Scanner::ScanFlowStart()
89 // flows can be simple keys
90 InsertPotentialSimpleKey();
91 m_simpleKeyAllowed = true;
92 m_canBeJSONFlow = false;
94 // eat
95 Mark mark = INPUT.mark();
96 char ch = INPUT.get();
97 FLOW_MARKER flowType = (ch == Keys::FlowSeqStart ? FLOW_SEQ : FLOW_MAP);
98 m_flows.push(flowType);
99 Token::TYPE type = (flowType == FLOW_SEQ ? Token::FLOW_SEQ_START : Token::FLOW_MAP_START);
100 m_tokens.push(Token(type, mark));
103 // FlowEnd
104 void Scanner::ScanFlowEnd()
106 if(InBlockContext())
107 throw ParserException(INPUT.mark(), ErrorMsg::FLOW_END);
109 // we might have a solo entry in the flow context
110 if(InFlowContext()) {
111 if(m_flows.top() == FLOW_MAP && VerifySimpleKey())
112 m_tokens.push(Token(Token::VALUE, INPUT.mark()));
113 else if(m_flows.top() == FLOW_SEQ)
114 InvalidateSimpleKey();
117 m_simpleKeyAllowed = false;
118 m_canBeJSONFlow = true;
120 // eat
121 Mark mark = INPUT.mark();
122 char ch = INPUT.get();
124 // check that it matches the start
125 FLOW_MARKER flowType = (ch == Keys::FlowSeqEnd ? FLOW_SEQ : FLOW_MAP);
126 if(m_flows.top() != flowType)
127 throw ParserException(mark, ErrorMsg::FLOW_END);
128 m_flows.pop();
130 Token::TYPE type = (flowType ? Token::FLOW_SEQ_END : Token::FLOW_MAP_END);
131 m_tokens.push(Token(type, mark));
134 // FlowEntry
135 void Scanner::ScanFlowEntry()
137 // we might have a solo entry in the flow context
138 if(InFlowContext()) {
139 if(m_flows.top() == FLOW_MAP && VerifySimpleKey())
140 m_tokens.push(Token(Token::VALUE, INPUT.mark()));
141 else if(m_flows.top() == FLOW_SEQ)
142 InvalidateSimpleKey();
145 m_simpleKeyAllowed = true;
146 m_canBeJSONFlow = false;
148 // eat
149 Mark mark = INPUT.mark();
150 INPUT.eat(1);
151 m_tokens.push(Token(Token::FLOW_ENTRY, mark));
154 // BlockEntry
155 void Scanner::ScanBlockEntry()
157 // we better be in the block context!
158 if(InFlowContext())
159 throw ParserException(INPUT.mark(), ErrorMsg::BLOCK_ENTRY);
161 // can we put it here?
162 if(!m_simpleKeyAllowed)
163 throw ParserException(INPUT.mark(), ErrorMsg::BLOCK_ENTRY);
165 PushIndentTo(INPUT.column(), IndentMarker::SEQ);
166 m_simpleKeyAllowed = true;
167 m_canBeJSONFlow = false;
169 // eat
170 Mark mark = INPUT.mark();
171 INPUT.eat(1);
172 m_tokens.push(Token(Token::BLOCK_ENTRY, mark));
175 // Key
176 void Scanner::ScanKey()
178 // handle keys diffently in the block context (and manage indents)
179 if(InBlockContext()) {
180 if(!m_simpleKeyAllowed)
181 throw ParserException(INPUT.mark(), ErrorMsg::MAP_KEY);
183 PushIndentTo(INPUT.column(), IndentMarker::MAP);
186 // can only put a simple key here if we're in block context
187 m_simpleKeyAllowed = InBlockContext();
189 // eat
190 Mark mark = INPUT.mark();
191 INPUT.eat(1);
192 m_tokens.push(Token(Token::KEY, mark));
195 // Value
196 void Scanner::ScanValue()
198 // and check that simple key
199 bool isSimpleKey = VerifySimpleKey();
200 m_canBeJSONFlow = false;
202 if(isSimpleKey) {
203 // can't follow a simple key with another simple key (dunno why, though - it seems fine)
204 m_simpleKeyAllowed = false;
205 } else {
206 // handle values diffently in the block context (and manage indents)
207 if(InBlockContext()) {
208 if(!m_simpleKeyAllowed)
209 throw ParserException(INPUT.mark(), ErrorMsg::MAP_VALUE);
211 PushIndentTo(INPUT.column(), IndentMarker::MAP);
214 // can only put a simple key here if we're in block context
215 m_simpleKeyAllowed = InBlockContext();
218 // eat
219 Mark mark = INPUT.mark();
220 INPUT.eat(1);
221 m_tokens.push(Token(Token::VALUE, mark));
224 // AnchorOrAlias
225 void Scanner::ScanAnchorOrAlias()
227 bool alias;
228 std::string name;
230 // insert a potential simple key
231 InsertPotentialSimpleKey();
232 m_simpleKeyAllowed = false;
233 m_canBeJSONFlow = false;
235 // eat the indicator
236 Mark mark = INPUT.mark();
237 char indicator = INPUT.get();
238 alias = (indicator == Keys::Alias);
240 // now eat the content
241 while(INPUT && Exp::Anchor().Matches(INPUT))
242 name += INPUT.get();
244 // we need to have read SOMETHING!
245 if(name.empty())
246 throw ParserException(INPUT.mark(), alias ? ErrorMsg::ALIAS_NOT_FOUND : ErrorMsg::ANCHOR_NOT_FOUND);
248 // and needs to end correctly
249 if(INPUT && !Exp::AnchorEnd().Matches(INPUT))
250 throw ParserException(INPUT.mark(), alias ? ErrorMsg::CHAR_IN_ALIAS : ErrorMsg::CHAR_IN_ANCHOR);
252 // and we're done
253 Token token(alias ? Token::ALIAS : Token::ANCHOR, mark);
254 token.value = name;
255 m_tokens.push(token);
258 // Tag
259 void Scanner::ScanTag()
261 // insert a potential simple key
262 InsertPotentialSimpleKey();
263 m_simpleKeyAllowed = false;
264 m_canBeJSONFlow = false;
266 Token token(Token::TAG, INPUT.mark());
268 // eat the indicator
269 INPUT.get();
271 if(INPUT && INPUT.peek() == Keys::VerbatimTagStart){
272 std::string tag = ScanVerbatimTag(INPUT);
274 token.value = tag;
275 token.data = Tag::VERBATIM;
276 } else {
277 bool canBeHandle;
278 token.value = ScanTagHandle(INPUT, canBeHandle);
279 if(!canBeHandle && token.value.empty())
280 token.data = Tag::NON_SPECIFIC;
281 else if(token.value.empty())
282 token.data = Tag::SECONDARY_HANDLE;
283 else
284 token.data = Tag::PRIMARY_HANDLE;
286 // is there a suffix?
287 if(canBeHandle && INPUT.peek() == Keys::Tag) {
288 // eat the indicator
289 INPUT.get();
290 token.params.push_back(ScanTagSuffix(INPUT));
291 token.data = Tag::NAMED_HANDLE;
295 m_tokens.push(token);
298 // PlainScalar
299 void Scanner::ScanPlainScalar()
301 std::string scalar;
303 // set up the scanning parameters
304 ScanScalarParams params;
305 params.end = (InFlowContext() ? Exp::EndScalarInFlow() : Exp::EndScalar()) || (Exp::BlankOrBreak() + Exp::Comment());
306 params.eatEnd = false;
307 params.indent = (InFlowContext() ? 0 : GetTopIndent() + 1);
308 params.fold = FOLD_FLOW;
309 params.eatLeadingWhitespace = true;
310 params.trimTrailingSpaces = true;
311 params.chomp = STRIP;
312 params.onDocIndicator = BREAK;
313 params.onTabInIndentation = THROW;
315 // insert a potential simple key
316 InsertPotentialSimpleKey();
318 Mark mark = INPUT.mark();
319 scalar = ScanScalar(INPUT, params);
321 // can have a simple key only if we ended the scalar by starting a new line
322 m_simpleKeyAllowed = params.leadingSpaces;
323 m_canBeJSONFlow = false;
325 // finally, check and see if we ended on an illegal character
326 //if(Exp::IllegalCharInScalar.Matches(INPUT))
327 // throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_SCALAR);
329 Token token(Token::PLAIN_SCALAR, mark);
330 token.value = scalar;
331 m_tokens.push(token);
334 // QuotedScalar
335 void Scanner::ScanQuotedScalar()
337 std::string scalar;
339 // peek at single or double quote (don't eat because we need to preserve (for the time being) the input position)
340 char quote = INPUT.peek();
341 bool single = (quote == '\'');
343 // setup the scanning parameters
344 ScanScalarParams params;
345 params.end = (single ? RegEx(quote) && !Exp::EscSingleQuote() : RegEx(quote));
346 params.eatEnd = true;
347 params.escape = (single ? '\'' : '\\');
348 params.indent = 0;
349 params.fold = FOLD_FLOW;
350 params.eatLeadingWhitespace = true;
351 params.trimTrailingSpaces = false;
352 params.chomp = CLIP;
353 params.onDocIndicator = THROW;
355 // insert a potential simple key
356 InsertPotentialSimpleKey();
358 Mark mark = INPUT.mark();
360 // now eat that opening quote
361 INPUT.get();
363 // and scan
364 scalar = ScanScalar(INPUT, params);
365 m_simpleKeyAllowed = false;
366 m_canBeJSONFlow = true;
368 Token token(Token::NON_PLAIN_SCALAR, mark);
369 token.value = scalar;
370 m_tokens.push(token);
373 // BlockScalarToken
374 // . These need a little extra processing beforehand.
375 // . We need to scan the line where the indicator is (this doesn't count as part of the scalar),
376 // and then we need to figure out what level of indentation we'll be using.
377 void Scanner::ScanBlockScalar()
379 std::string scalar;
381 ScanScalarParams params;
382 params.indent = 1;
383 params.detectIndent = true;
385 // eat block indicator ('|' or '>')
386 Mark mark = INPUT.mark();
387 char indicator = INPUT.get();
388 params.fold = (indicator == Keys::FoldedScalar ? FOLD_BLOCK : DONT_FOLD);
390 // eat chomping/indentation indicators
391 params.chomp = CLIP;
392 int n = Exp::Chomp().Match(INPUT);
393 for(int i=0;i<n;i++) {
394 char ch = INPUT.get();
395 if(ch == '+')
396 params.chomp = KEEP;
397 else if(ch == '-')
398 params.chomp = STRIP;
399 else if(Exp::Digit().Matches(ch)) {
400 if(ch == '0')
401 throw ParserException(INPUT.mark(), ErrorMsg::ZERO_INDENT_IN_BLOCK);
403 params.indent = ch - '0';
404 params.detectIndent = false;
408 // now eat whitespace
409 while(Exp::Blank().Matches(INPUT))
410 INPUT.eat(1);
412 // and comments to the end of the line
413 if(Exp::Comment().Matches(INPUT))
414 while(INPUT && !Exp::Break().Matches(INPUT))
415 INPUT.eat(1);
417 // if it's not a line break, then we ran into a bad character inline
418 if(INPUT && !Exp::Break().Matches(INPUT))
419 throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_BLOCK);
421 // set the initial indentation
422 if(GetTopIndent() >= 0)
423 params.indent += GetTopIndent();
425 params.eatLeadingWhitespace = false;
426 params.trimTrailingSpaces = false;
427 params.onTabInIndentation = THROW;
429 scalar = ScanScalar(INPUT, params);
431 // simple keys always ok after block scalars (since we're gonna start a new line anyways)
432 m_simpleKeyAllowed = true;
433 m_canBeJSONFlow = false;
435 Token token(Token::NON_PLAIN_SCALAR, mark);
436 token.value = scalar;
437 m_tokens.push(token);