From 3e3ce1443520ece1e3468f9d16d003cffcda9824 Mon Sep 17 00:00:00 2001 From: xi Date: Mon, 3 Apr 2006 18:20:25 +0000 Subject: [PATCH] Add style information to events generated by Parser. git-svn-id: http://svn.pyyaml.org/pyyaml/trunk@130 18f92427-320e-0410-9341-c67f048884a3 --- examples/yaml-hl/yaml_hl_ascii.cfg | 7 ++-- examples/yaml-hl/yaml_hl_html.cfg | 7 ++-- lib/yaml/composer.py | 8 ++-- lib/yaml/events.py | 77 ++++++++++++++++++++++++----------- lib/yaml/parser.py | 83 +++++++++++++++++++++++++------------- lib/yaml/reader.py | 4 ++ lib/yaml/scanner.py | 41 ++++++++++++------- lib/yaml/tokens.py | 10 ++++- tests/data/aliases.events | 8 ++++ tests/data/documents.events | 11 +++++ tests/data/mappings.events | 44 ++++++++++++++++++++ tests/data/scalars.events | 24 +++++++++++ tests/data/sequences.events | 81 +++++++++++++++++++++++++++++++++++++ tests/data/spec-08-12.canonical | 1 + tests/data/spec-08-12.data | 2 +- tests/test_appliance.py | 8 ++-- tests/test_emitter.py | 73 +++++++++++++++++++++++++++++++++ tests/test_structure.py | 12 +++--- tests/test_yaml.py | 1 + 19 files changed, 415 insertions(+), 87 deletions(-) create mode 100644 tests/data/aliases.events create mode 100644 tests/data/documents.events create mode 100644 tests/data/mappings.events create mode 100644 tests/data/scalars.events create mode 100644 tests/data/sequences.events create mode 100644 tests/test_emitter.py diff --git a/examples/yaml-hl/yaml_hl_ascii.cfg b/examples/yaml-hl/yaml_hl_ascii.cfg index 851d0f8..122f696 100644 --- a/examples/yaml-hl/yaml_hl_ascii.cfg +++ b/examples/yaml-hl/yaml_hl_ascii.cfg @@ -27,9 +27,10 @@ events: stream-end: { end: "\e[0m" } document-start: document-end: - sequence: - mapping: - collection-end: + sequence-start: + sequence-end: + mapping-start: + mapping-end: scalar: replaces: !!pairs diff --git a/examples/yaml-hl/yaml_hl_html.cfg b/examples/yaml-hl/yaml_hl_html.cfg index 903810e..5eab7af 100644 --- a/examples/yaml-hl/yaml_hl_html.cfg +++ b/examples/yaml-hl/yaml_hl_html.cfg @@ -26,9 +26,10 @@ events: stream-end: { end: } document-start: { start: } document-end: { end: } - sequence: { start: } - mapping: { start: } - collection-end: { end: } + sequence-start: { start: } + sequence-end: { end: } + mapping-start: { start: } + mapping-end: { end: } scalar: { start: , end: } replaces: !!pairs diff --git a/lib/yaml/composer.py b/lib/yaml/composer.py index 9b8c1e2..7152cbd 100644 --- a/lib/yaml/composer.py +++ b/lib/yaml/composer.py @@ -71,9 +71,9 @@ class Composer: self.all_anchors[anchor] = event if self.parser.check(ScalarEvent): node = self.compose_scalar_node() - elif self.parser.check(SequenceEvent): + elif self.parser.check(SequenceStartEvent): node = self.compose_sequence_node() - elif self.parser.check(MappingEvent): + elif self.parser.check(MappingStartEvent): node = self.compose_mapping_node() if anchor is not None: self.complete_anchors[anchor] = node @@ -87,7 +87,7 @@ class Composer: def compose_sequence_node(self): start_event = self.parser.get() value = [] - while not self.parser.check(CollectionEndEvent): + while not self.parser.check(SequenceEndEvent): value.append(self.compose_node()) end_event = self.parser.get() return SequenceNode(start_event.tag, value, @@ -96,7 +96,7 @@ class Composer: def compose_mapping_node(self): start_event = self.parser.get() value = {} - while not self.parser.check(CollectionEndEvent): + while not self.parser.check(MappingEndEvent): key_event = self.parser.peek() item_key = self.compose_node() item_value = self.compose_node() diff --git a/lib/yaml/events.py b/lib/yaml/events.py index 8837633..f9a695f 100644 --- a/lib/yaml/events.py +++ b/lib/yaml/events.py @@ -1,58 +1,89 @@ +# Abstract classes. + class Event: - def __init__(self, start_mark, end_mark): + def __init__(self, start_mark=None, end_mark=None): self.start_mark = start_mark self.end_mark = end_mark def __repr__(self): - attributes = [key for key in self.__dict__ - if not key.endswith('_mark')] - attributes.sort() + attributes = [key for key in ['anchor', 'tag', 'value'] + if hasattr(self, key)] arguments = ', '.join(['%s=%r' % (key, getattr(self, key)) for key in attributes]) return '%s(%s)' % (self.__class__.__name__, arguments) class NodeEvent(Event): - def __init__(self, anchor, start_mark, end_mark): + def __init__(self, anchor, start_mark=None, end_mark=None): self.anchor = anchor self.start_mark = start_mark self.end_mark = end_mark -class AliasEvent(NodeEvent): - pass - -class ScalarEvent(NodeEvent): - def __init__(self, anchor, tag, value, start_mark, end_mark): +class CollectionStartEvent(NodeEvent): + def __init__(self, anchor, tag, start_mark=None, end_mark=None, + flow_style=None): self.anchor = anchor self.tag = tag - self.value = value self.start_mark = start_mark self.end_mark = end_mark + self.flow_style = flow_style -class CollectionEvent(NodeEvent): - def __init__(self, anchor, tag, start_mark, end_mark): - self.anchor = anchor - self.tag = tag +class CollectionEndEvent(Event): + pass + +# Implementations. + +class StreamStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + encoding=None, canonical=None, indent=None, width=None): self.start_mark = start_mark self.end_mark = end_mark + self.encoding = encoding + self.canonical = canonical + self.indent = indent + self.width = width -class SequenceEvent(CollectionEvent): +class StreamEndEvent(Event): pass -class MappingEvent(CollectionEvent): - pass +class DocumentStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + implicit=None, version=None, tags=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.implicit = implicit + self.version = version + self.tags = tags -class CollectionEndEvent(Event): +class DocumentEndEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + implicit=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.implicit = implicit + +class AliasEvent(NodeEvent): pass -class DocumentStartEvent(Event): +class ScalarEvent(NodeEvent): + def __init__(self, anchor, tag, value, start_mark=None, end_mark=None, + implicit=None, style=None): + self.anchor = anchor + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.implicit = implicit + self.style = style + +class SequenceStartEvent(CollectionStartEvent): pass -class DocumentEndEvent(Event): +class SequenceEndEvent(CollectionEndEvent): pass -class StreamStartEvent(Event): +class MappingStartEvent(CollectionStartEvent): pass -class StreamEndEvent(Event): +class MappingEndEvent(CollectionEndEvent): pass diff --git a/lib/yaml/parser.py b/lib/yaml/parser.py index 9870699..4e0db9f 100644 --- a/lib/yaml/parser.py +++ b/lib/yaml/parser.py @@ -130,7 +130,8 @@ class Parser: # Parse start of stream. token = self.scanner.get() - yield StreamStartEvent(token.start_mark, token.end_mark) + yield StreamStartEvent(token.start_mark, token.end_mark, + encoding=token.encoding) # Parse implicit document. if not self.scanner.check(DirectiveToken, DocumentStartToken, @@ -138,21 +139,25 @@ class Parser: self.tag_handles = self.DEFAULT_TAGS token = self.scanner.peek() start_mark = end_mark = token.start_mark - yield DocumentStartEvent(start_mark, end_mark) + yield DocumentStartEvent(start_mark, end_mark, + implicit=True) for event in self.parse_block_node(): yield event token = self.scanner.peek() start_mark = end_mark = token.start_mark + implicit = True while self.scanner.check(DocumentEndToken): token = self.scanner.get() end_mark = token.end_mark - yield DocumentEndEvent(start_mark, end_mark) + implicit = True + yield DocumentEndEvent(start_mark, end_mark, + implicit=implicit) # Parse explicit documents. while not self.scanner.check(StreamEndToken): token = self.scanner.peek() start_mark = token.start_mark - self.process_directives() + version, tags = self.process_directives() if not self.scanner.check(DocumentStartToken): raise ParserError(None, None, "expected '', but found %r" @@ -160,7 +165,8 @@ class Parser: self.scanner.peek().start_mark) token = self.scanner.get() end_mark = token.end_mark - yield DocumentStartEvent(start_mark, end_mark) + yield DocumentStartEvent(start_mark, end_mark, + implicit=False, version=version, tags=tags) if self.scanner.check(DirectiveToken, DocumentStartToken, DocumentEndToken, StreamEndToken): yield self.process_empty_scalar(token.end_mark) @@ -169,10 +175,13 @@ class Parser: yield event token = self.scanner.peek() start_mark = end_mark = token.start_mark + implicit=True while self.scanner.check(DocumentEndToken): token = self.scanner.get() end_mark = token.end_mark - yield DocumentEndEvent(start_mark, end_mark) + implicit=False + yield DocumentEndEvent(start_mark, end_mark, + implicit=implicit) # Parse end of stream. token = self.scanner.get() @@ -201,9 +210,14 @@ class Parser: "duplicate tag handle %r" % handle.encode('utf-8'), token.start_mark) self.tag_handles[handle] = prefix + if self.tag_handles: + value = self.yaml_version, self.tag_handles.copy() + else: + value = self.yaml_version, None for key in self.DEFAULT_TAGS: if key not in self.tag_handles: self.tag_handles[key] = self.DEFAULT_TAGS[key] + return value def parse_block_node(self): return self.parse_node(block=True) @@ -232,19 +246,22 @@ class Parser: start_mark = end_mark = tag_mark = None if self.scanner.check(AnchorToken): token = self.scanner.get() - start_mark = end_mark = token.start_mark + start_mark = token.start_mark + end_mark = token.end_mark anchor = token.value if self.scanner.check(TagToken): token = self.scanner.get() - end_mark = tag_mark = token.start_mark + tag_mark = token.start_mark + end_mark = token.end_mark tag = token.value elif self.scanner.check(TagToken): token = self.scanner.get() - start_mark = end_mark = tag_mark = token.start_mark + start_mark = tag_mark = token.start_mark + end_mark = token.end_mark tag = token.value if self.scanner.check(AnchorToken): token = self.scanner.get() - end_mark = token.start_mark + end_mark = token.end_mark anchor = token.value if tag is not None: handle, suffix = tag @@ -258,38 +275,48 @@ class Parser: tag = suffix if tag is None: if not (self.scanner.check(ScalarToken) and - self.scanner.peek().plain): + self.scanner.peek().implicit): tag = u'!' if start_mark is None: - start_mark = self.scanner.peek().start_mark + start_mark = end_mark = self.scanner.peek().start_mark event = None collection_events = None if indentless_sequence and self.scanner.check(BlockEntryToken): end_mark = self.scanner.peek().end_mark - event = SequenceEvent(anchor, tag, start_mark, end_mark) + event = SequenceStartEvent(anchor, tag, start_mark, end_mark) collection_events = self.parse_indentless_sequence() else: if self.scanner.check(ScalarToken): token = self.scanner.get() end_mark = token.end_mark event = ScalarEvent(anchor, tag, token.value, - start_mark, end_mark) + start_mark, end_mark, + implicit=token.implicit, style=token.style) elif self.scanner.check(FlowSequenceStartToken): end_mark = self.scanner.peek().end_mark - event = SequenceEvent(anchor, tag, start_mark, end_mark) + event = SequenceStartEvent(anchor, tag, start_mark, end_mark, + flow_style=True) collection_events = self.parse_flow_sequence() elif self.scanner.check(FlowMappingStartToken): end_mark = self.scanner.peek().end_mark - event = MappingEvent(anchor, tag, start_mark, end_mark) + event = MappingStartEvent(anchor, tag, start_mark, end_mark, + flow_style=True) collection_events = self.parse_flow_mapping() elif block and self.scanner.check(BlockSequenceStartToken): end_mark = self.scanner.peek().start_mark - event = SequenceEvent(anchor, tag, start_mark, end_mark) + event = SequenceStartEvent(anchor, tag, start_mark, end_mark, + flow_style=False) collection_events = self.parse_block_sequence() elif block and self.scanner.check(BlockMappingStartToken): end_mark = self.scanner.peek().start_mark - event = MappingEvent(anchor, tag, start_mark, end_mark) + event = MappingStartEvent(anchor, tag, start_mark, end_mark, + flow_style=False) collection_events = self.parse_block_mapping() + elif anchor is not None or tag is not None: + # Empty scalars are allowed even if a tag or an anchor is + # specified. + event = ScalarEvent(anchor, tag, u'', start_mark, end_mark, + implicit=True) else: if block: node = 'block' @@ -320,7 +347,7 @@ class Parser: raise ParserError("while scanning a block collection", start_mark, "expected , but found %r" % token.id, token.start_mark) token = self.scanner.get() - yield CollectionEndEvent(token.start_mark, token.end_mark) + yield SequenceEndEvent(token.start_mark, token.end_mark) def parse_indentless_sequence(self): # (BLOCK-ENTRY block_node?)+ @@ -333,7 +360,7 @@ class Parser: else: yield self.process_empty_scalar(token.end_mark) token = self.scanner.peek() - yield CollectionEndEvent(token.start_mark, token.start_mark) + yield SequenceEndEvent(token.start_mark, token.start_mark) def parse_block_mapping(self): # BLOCK-MAPPING_START @@ -365,7 +392,7 @@ class Parser: raise ParserError("while scanning a block mapping", start_mark, "expected , but found %r" % token.id, token.start_mark) token = self.scanner.get() - yield CollectionEndEvent(token.start_mark, token.end_mark) + yield MappingEndEvent(token.start_mark, token.end_mark) def parse_flow_sequence(self): # flow_sequence ::= FLOW-SEQUENCE-START @@ -383,8 +410,9 @@ class Parser: while not self.scanner.check(FlowSequenceEndToken): if self.scanner.check(KeyToken): token = self.scanner.get() - yield MappingEvent(None, u'!', - token.start_mark, token.end_mark) + yield MappingStartEvent(None, u'!', + token.start_mark, token.end_mark, + flow_style=True) if not self.scanner.check(ValueToken, FlowEntryToken, FlowSequenceEndToken): for event in self.parse_flow_node(): @@ -402,7 +430,7 @@ class Parser: token = self.scanner.peek() yield self.process_empty_scalar(token.start_mark) token = self.scanner.peek() - yield CollectionEndEvent(token.start_mark, token.start_mark) + yield MappingEndEvent(token.start_mark, token.start_mark) else: for event in self.parse_flow_node(): yield event @@ -413,7 +441,7 @@ class Parser: if self.scanner.check(FlowEntryToken): self.scanner.get() token = self.scanner.get() - yield CollectionEndEvent(token.start_mark, token.end_mark) + yield SequenceEndEvent(token.start_mark, token.end_mark) def parse_flow_mapping(self): # flow_mapping ::= FLOW-MAPPING-START @@ -457,8 +485,9 @@ class Parser: raise ParserError("while scanning a flow mapping", start_mark, "expected '}', but found %r" % token.id, token.start_mark) token = self.scanner.get() - yield CollectionEndEvent(token.start_mark, token.end_mark) + yield MappingEndEvent(token.start_mark, token.end_mark) def process_empty_scalar(self, mark): - return ScalarEvent(None, None, u'', mark, mark) + return ScalarEvent(None, None, u'', mark, mark, + implicit=True) diff --git a/lib/yaml/reader.py b/lib/yaml/reader.py index 9778943..ab16a13 100644 --- a/lib/yaml/reader.py +++ b/lib/yaml/reader.py @@ -100,6 +100,7 @@ class Reader: self.pointer = 0 self.raw_buffer = None self.raw_decode = None + self.encoding = None self.index = 0 self.line = 0 self.column = 0 @@ -156,10 +157,13 @@ class Reader: if not isinstance(self.raw_buffer, unicode): if self.raw_buffer.startswith(codecs.BOM_UTF16_LE): self.raw_decode = utf_16_le_decode + self.encoding = 'utf-16-le' elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE): self.raw_decode = utf_16_be_decode + self.encoding = 'utf-16-be' else: self.raw_decode = utf_8_decode + self.encoding = 'utf-8' self.update(1) NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD]') diff --git a/lib/yaml/scanner.py b/lib/yaml/scanner.py index 9c536b4..ff4a8cc 100644 --- a/lib/yaml/scanner.py +++ b/lib/yaml/scanner.py @@ -379,8 +379,9 @@ class Scanner: # Read the token. mark = self.reader.get_mark() - # Add STREAM-END. - self.tokens.append(StreamStartToken(mark, mark)) + # Add STREAM-START. + self.tokens.append(StreamStartToken(mark, mark, + encoding=self.reader.encoding)) def fetch_stream_end(self): @@ -638,12 +639,12 @@ class Scanner: self.tokens.append(self.scan_tag()) def fetch_literal(self): - self.fetch_block_scalar(folded=False) + self.fetch_block_scalar(style='|') def fetch_folded(self): - self.fetch_block_scalar(folded=True) + self.fetch_block_scalar(style='>') - def fetch_block_scalar(self, folded): + def fetch_block_scalar(self, style): # A simple key may follow a block scalar. self.allow_simple_key = True @@ -652,15 +653,15 @@ class Scanner: self.remove_possible_simple_key() # Scan and add SCALAR. - self.tokens.append(self.scan_block_scalar(folded)) + self.tokens.append(self.scan_block_scalar(style)) def fetch_single(self): - self.fetch_flow_scalar(double=False) + self.fetch_flow_scalar(style='\'') def fetch_double(self): - self.fetch_flow_scalar(double=True) + self.fetch_flow_scalar(style='"') - def fetch_flow_scalar(self, double): + def fetch_flow_scalar(self, style): # A flow scalar could be a simple key. self.save_possible_simple_key() @@ -669,7 +670,7 @@ class Scanner: self.allow_simple_key = False # Scan and add SCALAR. - self.tokens.append(self.scan_flow_scalar(double)) + self.tokens.append(self.scan_flow_scalar(style)) def fetch_plain(self): @@ -986,9 +987,14 @@ class Scanner: end_mark = self.reader.get_mark() return TagToken(value, start_mark, end_mark) - def scan_block_scalar(self, folded): + def scan_block_scalar(self, style): # See the specification for details. + if style == '>': + folded = True + else: + folded = False + chunks = [] start_mark = self.reader.get_mark() @@ -1021,6 +1027,7 @@ class Scanner: line_break = self.scan_line_break() breaks, end_mark = self.scan_block_scalar_breaks(indent) if self.reader.column == indent and self.reader.peek() != u'\0': + # Unfortunately, folding rules are ambiguous. # # This is the folding according to the specification: @@ -1053,7 +1060,8 @@ class Scanner: chunks.extend(breaks) # We are done. - return ScalarToken(u''.join(chunks), False, start_mark, end_mark) + return ScalarToken(u''.join(chunks), False, start_mark, end_mark, + style) def scan_block_scalar_indicators(self, start_mark): # See the specification for details. @@ -1137,13 +1145,17 @@ class Scanner: self.reader.forward() return chunks, end_mark - def scan_flow_scalar(self, double): + def scan_flow_scalar(self, style): # See the specification for details. # Note that we loose indentation rules for quoted scalars. Quoted # scalars don't need to adhere indentation because " and ' clearly # mark the beginning and the end of them. Therefore we are less # restrictive then the specification requires. We only need to check # that document separators are not included in scalars. + if style == '"': + double = True + else: + double = False chunks = [] start_mark = self.reader.get_mark() quote = self.reader.peek() @@ -1154,7 +1166,8 @@ class Scanner: chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark)) self.reader.forward() end_mark = self.reader.get_mark() - return ScalarToken(u''.join(chunks), False, start_mark, end_mark) + return ScalarToken(u''.join(chunks), False, start_mark, end_mark, + style) ESCAPE_REPLACEMENTS = { u'0': u'\0', diff --git a/lib/yaml/tokens.py b/lib/yaml/tokens.py index 59b36af..933e56f 100644 --- a/lib/yaml/tokens.py +++ b/lib/yaml/tokens.py @@ -30,6 +30,11 @@ class DocumentEndToken(Token): class StreamStartToken(Token): id = '' + def __init__(self, start_mark=None, end_mark=None, + encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding class StreamEndToken(Token): id = '' @@ -90,9 +95,10 @@ class TagToken(Token): class ScalarToken(Token): id = '' - def __init__(self, value, plain, start_mark, end_mark): + def __init__(self, value, implicit, start_mark, end_mark, style=None): self.value = value - self.plain = plain + self.implicit = implicit self.start_mark = start_mark self.end_mark = end_mark + self.style = style diff --git a/tests/data/aliases.events b/tests/data/aliases.events new file mode 100644 index 0000000..9139b51 --- /dev/null +++ b/tests/data/aliases.events @@ -0,0 +1,8 @@ +- !StreamStart +- !DocumentStart +- !SequenceStart +- !Scalar { anchor: 'myanchor', tag: '!mytag', value: 'data' } +- !Alias { anchor: 'myanchor' } +- !SequenceEnd +- !DocumentEnd +- !StreamEnd diff --git a/tests/data/documents.events b/tests/data/documents.events new file mode 100644 index 0000000..223a314 --- /dev/null +++ b/tests/data/documents.events @@ -0,0 +1,11 @@ +- !StreamStart +- !DocumentStart +- !Scalar { implicit: true } +- !DocumentEnd +- !DocumentStart +- !Scalar { implicit: true } +- !DocumentEnd +- !DocumentStart +- !Scalar { implicit: true } +- !DocumentEnd +- !StreamEnd diff --git a/tests/data/mappings.events b/tests/data/mappings.events new file mode 100644 index 0000000..e66ec09 --- /dev/null +++ b/tests/data/mappings.events @@ -0,0 +1,44 @@ +- !StreamStart + +- !DocumentStart +- !MappingStart +- !Scalar { implicit: true, value: 'key' } +- !Scalar { implicit: true, value: 'value' } +- !Scalar { implicit: true, value: 'empty mapping' } +- !MappingStart +- !MappingEnd +- !Scalar { implicit: true, value: 'empty mapping with tag' } +- !MappingStart { tag: '!mytag' } +- !MappingEnd +- !Scalar { implicit: true, value: 'block mapping' } +- !MappingStart +- !MappingStart +- !Scalar { implicit: true, value: 'complex' } +- !Scalar { implicit: true, value: 'key' } +- !Scalar { implicit: true, value: 'complex' } +- !Scalar { implicit: true, value: 'key' } +- !MappingEnd +- !MappingStart +- !Scalar { implicit: true, value: 'complex' } +- !Scalar { implicit: true, value: 'key' } +- !MappingEnd +- !MappingEnd +- !Scalar { implicit: true, value: 'flow mapping' } +- !MappingStart { flow: true } +- !Scalar { implicit: true, value: 'key' } +- !Scalar { implicit: true, value: 'value' } +- !MappingStart +- !Scalar { implicit: true, value: 'complex' } +- !Scalar { implicit: true, value: 'key' } +- !Scalar { implicit: true, value: 'complex' } +- !Scalar { implicit: true, value: 'key' } +- !MappingEnd +- !MappingStart +- !Scalar { implicit: true, value: 'complex' } +- !Scalar { implicit: true, value: 'key' } +- !MappingEnd +- !MappingEnd +- !MappingEnd +- !DocumentEnd + +- !StreamEnd diff --git a/tests/data/scalars.events b/tests/data/scalars.events new file mode 100644 index 0000000..0d3f024 --- /dev/null +++ b/tests/data/scalars.events @@ -0,0 +1,24 @@ +- !StreamStart + +- !DocumentStart +- !MappingStart +- !Scalar { implicit: true, value: 'empty scalar' } +- !Scalar { implicit: true, value: '' } +- !Scalar { implicit: true, value: 'implicit scalar' } +- !Scalar { implicit: true, value: 'data' } +- !Scalar { implicit: true, value: 'quoted scalar' } +- !Scalar { value: 'data', style: '"' } +- !Scalar { implicit: true, value: 'block scalar' } +- !Scalar { value: 'data', style: '|' } +- !Scalar { implicit: true, value: 'empty scalar with tag' } +- !Scalar { implicit: true, tag: '!mytag', value: '' } +- !Scalar { implicit: true, value: 'implicit scalar with tag' } +- !Scalar { implicit: true, tag: '!mytag', value: 'data' } +- !Scalar { implicit: true, value: 'quoted scalar with tag' } +- !Scalar { value: 'data', style: '"', tag: '!mytag' } +- !Scalar { implicit: true, value: 'block scalar with tag' } +- !Scalar { value: 'data', style: '|', tag: '!mytag' } +- !MappingEnd +- !DocumentEnd + +- !StreamEnd diff --git a/tests/data/sequences.events b/tests/data/sequences.events new file mode 100644 index 0000000..0fb514e --- /dev/null +++ b/tests/data/sequences.events @@ -0,0 +1,81 @@ +- !StreamStart + +- !DocumentStart +- !SequenceStart +- !SequenceEnd +- !DocumentEnd + +- !DocumentStart +- !SequenceStart { tag: '!mytag' } +- !SequenceEnd +- !DocumentEnd + +- !DocumentStart +- !SequenceStart +- !SequenceStart +- !SequenceEnd +- !SequenceStart { tag: '!mytag' } +- !SequenceEnd +- !SequenceStart +- !Scalar +- !Scalar { value: 'data' } +- !Scalar { tag: '!mytag', value: 'data' } +- !SequenceEnd +- !SequenceStart +- !SequenceStart +- !SequenceStart +- !Scalar +- !SequenceEnd +- !SequenceEnd +- !SequenceEnd +- !SequenceStart +- !SequenceStart { tag: '!mytag' } +- !SequenceStart +- !Scalar { value: 'data' } +- !SequenceEnd +- !SequenceEnd +- !SequenceEnd +- !SequenceEnd +- !DocumentEnd + +- !DocumentStart +- !SequenceStart +- !MappingStart +- !Scalar { value: 'key1' } +- !SequenceStart +- !Scalar { value: 'data1' } +- !Scalar { value: 'data2' } +- !SequenceEnd +- !Scalar { value: 'key2' } +- !SequenceStart { tag: '!mytag1' } +- !Scalar { value: 'data3' } +- !SequenceStart +- !Scalar { value: 'data4' } +- !Scalar { value: 'data5' } +- !SequenceEnd +- !SequenceStart { tag: '!mytag2' } +- !Scalar { value: 'data6' } +- !Scalar { value: 'data7' } +- !SequenceEnd +- !SequenceEnd +- !MappingEnd +- !SequenceEnd +- !DocumentEnd + +- !DocumentStart +- !SequenceStart +- !SequenceStart { flow: true } +- !SequenceStart +- !SequenceEnd +- !Scalar +- !Scalar { value: 'data' } +- !Scalar { tag: '!mytag', value: 'data' } +- !SequenceStart { tag: '!mytag' } +- !Scalar { value: 'data' } +- !Scalar { value: 'data' } +- !SequenceEnd +- !SequenceEnd +- !SequenceEnd +- !DocumentEnd + +- !StreamEnd diff --git a/tests/data/spec-08-12.canonical b/tests/data/spec-08-12.canonical index dc3b81a..93899f4 100644 --- a/tests/data/spec-08-12.canonical +++ b/tests/data/spec-08-12.canonical @@ -6,4 +6,5 @@ !!str "Tagged", *A, !!str "", + !!str "", ] diff --git a/tests/data/spec-08-12.data b/tests/data/spec-08-12.data index d936ba2..3d4c6b7 100644 --- a/tests/data/spec-08-12.data +++ b/tests/data/spec-08-12.data @@ -3,6 +3,6 @@ &anchor "Anchored", !!str 'Tagged', *anchor, # Alias node -# !!str, # Empty plain scalar + !!str , # Empty plain scalar '', # Empty plain scalar ] diff --git a/tests/test_appliance.py b/tests/test_appliance.py index eb54faa..ed76e47 100644 --- a/tests/test_appliance.py +++ b/tests/test_appliance.py @@ -243,10 +243,10 @@ class CanonicalParser: if self.test_token(ScalarToken): self.events.append(ScalarEvent(anchor, tag, self.get_value(), None, None)) elif self.test_token(FlowSequenceStartToken): - self.events.append(SequenceEvent(anchor, tag, None, None)) + self.events.append(SequenceStartEvent(anchor, tag, None, None)) self.parse_sequence() elif self.test_token(FlowMappingStartToken): - self.events.append(MappingEvent(anchor, tag, None, None)) + self.events.append(MappingStartEvent(anchor, tag, None, None)) self.parse_mapping() else: raise Error("SCALAR, '[', or '{' is expected, got "+repr(self.tokens[self.index])) @@ -261,7 +261,7 @@ class CanonicalParser: if not self.test_token(FlowSequenceEndToken): self.parse_node() self.consume_token(FlowSequenceEndToken) - self.events.append(CollectionEndEvent(None, None)) + self.events.append(SequenceEndEvent(None, None)) # mapping: MAPPING-START (map_entry (ENTRY map_entry)*)? ENTRY? MAPPING-END def parse_mapping(self): @@ -273,7 +273,7 @@ class CanonicalParser: if not self.test_token(FlowMappingEndToken): self.parse_map_entry() self.consume_token(FlowMappingEndToken) - self.events.append(CollectionEndEvent(None, None)) + self.events.append(MappingEndEvent(None, None)) # map_entry: KEY node VALUE node def parse_map_entry(self): diff --git a/tests/test_emitter.py b/tests/test_emitter.py new file mode 100644 index 0000000..2da6696 --- /dev/null +++ b/tests/test_emitter.py @@ -0,0 +1,73 @@ + +import test_appliance, sys, StringIO + +from yaml import * +import yaml + +class TestEmitterOnCanonical(test_appliance.TestAppliance): + + def _testEmitterOnCanonical(self, test_name, canonical_filename): + events = list(iter(Parser(Scanner(Reader(file(canonical_filename, 'rb')))))) + #writer = sys.stdout + writer = StringIO.StringIO() + emitter = Emitter(writer) + #print "-"*30 + #print "ORIGINAL DATA:" + #print file(canonical_filename, 'rb').read() + for event in events: + emitter.emit(event) + data = writer.getvalue() + new_events = list(parse(data)) + self.failUnlessEqual(len(events), len(new_events)) + for event, new_event in zip(events, new_events): + self.failUnlessEqual(event.__class__, new_event.__class__) + +TestEmitterOnCanonical.add_tests('testEmitterOnCanonical', '.canonical') + +class EventsConstructor(Constructor): + + def construct_event(self, node): + if isinstance(node, ScalarNode): + mapping = {} + else: + mapping = self.construct_mapping(node) + class_name = str(node.tag[1:])+'Event' + if class_name in ['AliasEvent', 'ScalarEvent', 'SequenceStartEvent', 'MappingStartEvent']: + mapping.setdefault('anchor', None) + if class_name in ['ScalarEvent', 'SequenceStartEvent', 'MappingStartEvent']: + mapping.setdefault('tag', None) + if class_name == 'ScalarEvent': + mapping.setdefault('value', '') + value = getattr(yaml, class_name)(**mapping) + return value + +EventsConstructor.add_constructor(None, EventsConstructor.construct_event) + +class TestEmitter(test_appliance.TestAppliance): + + def _testEmitter(self, test_name, events_filename): + events = load_document(file(events_filename, 'rb'), Constructor=EventsConstructor) + self._dump(events_filename, events) + writer = StringIO.StringIO() + emitter = Emitter(writer) + for event in events: + emitter.emit(event) + data = writer.getvalue() + new_events = list(parse(data)) + self.failUnlessEqual(len(events), len(new_events)) + for event, new_event in zip(events, new_events): + self.failUnlessEqual(event.__class__, new_event.__class__) + + def _dump(self, events_filename, events): + writer = sys.stdout + emitter = Emitter(writer) + print "="*30 + print "EVENTS:" + print file(events_filename, 'rb').read() + print '-'*30 + print "OUTPUT:" + for event in events: + emitter.emit(event) + +TestEmitter.add_tests('testEmitter', '.events') + diff --git a/tests/test_structure.py b/tests/test_structure.py index ca3cb60..cb9a017 100644 --- a/tests/test_structure.py +++ b/tests/test_structure.py @@ -35,17 +35,17 @@ class TestStructure(test_appliance.TestAppliance): return True else: return None - elif parser.check(SequenceEvent): + elif parser.check(SequenceStartEvent): parser.get() sequence = [] - while not parser.check(CollectionEndEvent): + while not parser.check(SequenceEndEvent): sequence.append(self._convert(parser)) parser.get() return sequence - elif parser.check(MappingEvent): + elif parser.check(MappingStartEvent): parser.get() mapping = [] - while not parser.check(CollectionEndEvent): + while not parser.check(MappingEndEvent): key = self._convert(parser) value = self._convert(parser) mapping.append((key, value)) @@ -92,7 +92,7 @@ class TestParser(test_appliance.TestAppliance): #self.failUnlessEqual(event1.anchor, event2.anchor) #self.failUnlessEqual(event1.tag, event2.tag) self.failUnlessEqual(event1.value, event2.value) - if isinstance(event1, CollectionEvent): + if isinstance(event1, CollectionStartEvent): #self.failUnlessEqual(event1.anchor, event2.anchor) #self.failUnlessEqual(event1.tag, event2.tag) pass @@ -213,7 +213,7 @@ class TestParserOnCanonical(test_appliance.TestAppliance): self.failUnlessEqual(event1.anchor, event2.anchor) self.failUnlessEqual(event1.tag, event2.tag) self.failUnlessEqual(event1.value, event2.value) - if isinstance(event1, CollectionEvent): + if isinstance(event1, CollectionStartEvent): self.failUnlessEqual(event1.anchor, event2.anchor) self.failUnlessEqual(event1.tag, event2.tag) diff --git a/tests/test_yaml.py b/tests/test_yaml.py index bb5a9f1..a2759c4 100644 --- a/tests/test_yaml.py +++ b/tests/test_yaml.py @@ -9,6 +9,7 @@ from test_structure import * from test_errors import * from test_detector import * from test_constructor import * +#from test_emitter import * from test_syck import * def main(module='__main__'): -- 2.11.4.GIT