2 # Copyright (C) 2002-2003, International Business Machines Corporation and others.
8 # See Unicode Standard Annex #29.
9 # These rules are based on Version 4.0.0, dated 2003-04-17
14 ####################################################################################
16 # Character class definitions from TR 29
18 ####################################################################################
19 $Katakana = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
20 [:name = HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
21 [:name = HALFWIDTH KATAKANA VOICED SOUND MARK:]
22 [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]];
24 $Ideographic = [:Ideographic:];
25 $Hangul = [:Script = HANGUL:];
27 $ALetter = [\u0002 [:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:]
28 [:name = PERCENT SIGN:] [:name = PER MILLE SIGN:] [:name = PER TEN THOUSAND SIGN:]
29 [:name = SECTION SIGN:] [:name = DEGREE SIGN:] [:name = EURO SIGN:]
30 [:name = HYPHEN-MINUS:] [:name = EN DASH:] [:name = EM DASH:]
34 [:name = DIGIT THREE:]
38 [:name = DIGIT SEVEN:]
39 [:name = DIGIT EIGHT:]
46 - [:Script = Hiragana:]];
48 $MidLetter = [[:name = APOSTROPHE:] [:name = MIDDLE DOT:] [:name = HEBREW PUNCTUATION GERSHAYIM:]
49 [:name = RIGHT SINGLE QUOTATION MARK:] [:name = HYPHENATION POINT:]
50 [:name = HYPHEN-MINUS:] [:name = EURO SIGN:] [:name = PERCENT SIGN:]
51 [:name = PER MILLE SIGN:] [:name = PER TEN THOUSAND SIGN:]
52 [:name = EN DASH:] [:name = EM DASH:]
53 [:name = PERCENT SIGN:] [:name = SECTION SIGN:] [:name = DEGREE SIGN:]];
55 $MidNum = [[:LineBreak = Infix_Numeric:] - [:name = FULL STOP:]];
56 $Numeric = [:LineBreak = Numeric:];
62 # Character Class Definitions.
63 # The names are those from TR29.
67 $Control = [[[:Zl:] [:Zp:] [:Cc:] [:Cf:]] - $TheZWSP];
68 $Extend = [[:Grapheme_Extend = TRUE:]];
73 ####################################################################################
75 # Word Break Rules. Definitions and Rules specific to word break begin Here.
77 ####################################################################################
79 $Format = [[:Cf:] - $TheZWSP];
83 # Rule 3: Treat a grapheme cluster as if it were a single character.
84 # Hangul Syllables are easier to deal with here than they are in Grapheme Clusters
85 # because we don't need to find the boundaries between adjacent syllables -
86 # they won't be word boundaries.
91 # "Extended" definitions. Grapheme Cluster + Format Chars, treated like the base char.
93 $ALetterEx = $ALetter $Extend*;
94 $NumericEx = $Numeric $Extend*;
95 $MidNumEx = $MidNum $Extend*;
96 $MidLetterEx = $MidLetter $Extend*;
97 $KatakanaEx = $Katakana $Extend*;
98 $IdeographicEx= $Ideographic $Extend*;
99 $HangulEx = $Hangul $Extend*;
100 $FormatEx = $Format $Extend*;
104 # Numbers. Rules 8, 11, 12 form the TR.
106 $NumberSequence = $NumericEx ($FormatEx* $MidNumEx? $FormatEx* $NumericEx)*;
107 $NumberSequence {100};
110 # Words. Alpha-numerics. Rule 5, 6, 7, 9, 10
111 # - must include at least one letter.
112 # - may include both letters and numbers.
113 # - may include MideLetter, MidNumber punctuation.
115 $LetterSequence = $ALetterEx ($FormatEx* $MidLetterEx? $FormatEx* $ALetterEx)*; # rules #6, #7
116 ($NumberSequence $FormatEx*)? $LetterSequence ($FormatEx* ($NumberSequence | $LetterSequence))* {200};
118 # Punctuations by themselves
119 [[:P:][:S:]-[:name = FULL STOP:]]*;
120 [[:name = FULL STOP:]]*;
123 # Do not break between Katakana. Rule #13.
125 $KatakanaEx ($FormatEx* $KatakanaEx)* {300};
126 [:Hiragana:] $Extend* {300};
129 # Ideographic Characters. Stand by themselves as words.
130 # Separated from the "Everything Else" rule, below, only so that they
131 # can be tagged with a return value. TODO: is this what we want?
133 $IdeographicEx ($FormatEx* $IdeographicEx)* {400};
134 $HangulEx ($FormatEx* $HangulEx)* {400};
137 # Everything Else, with no tag.
138 # Non-Control chars combine with $Extend (combining) chars.
139 # Controls are do not.
141 [^$Control [:Ideographic:]] $Extend*;
145 # Reverse Rules. Back up over any of the chars that can group together.
146 # (Reverse rules do not need to be exact; they can back up too far,
147 # but must back up at least enough, and must stop on a boundary.)
150 # NonStarters are the set of all characters that can appear at the 2nd - nth position of
151 # a word. (They may also be the first.) The reverse rule skips over these, until it
152 # reaches something that can only be the start (and probably only) char in a "word".
153 # A space or punctuation meets the test.
155 $NonStarters = [$Numeric $ALetter $Katakana $Ideographic $Hangul [:P:] [:S:] $MidLetter $MidNum $Extend $Format];
158 ! ($NonStarters* | \n \r) .;