2 # Copyright (C) 2002-2003, International Business Machines Corporation and others.
8 # See Unicode Standard Annex #29.
9 # These rules are based on Version 4.0.0, dated 2003-04-17
14 ####################################################################################
16 # Character class definitions from TR 29
18 ####################################################################################
19 $Katakana = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
20 [:name = HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
21 [:name = HALFWIDTH KATAKANA VOICED SOUND MARK:]
22 [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]];
24 $Ideographic = [:Ideographic:];
25 $Hangul = [:Script = HANGUL:];
27 $ALetter = [\u0002 [:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:]
33 - [:Script = Hiragana:]];
35 $MidLetter = [[:name = APOSTROPHE:] [:name = MIDDLE DOT:] [:name = HEBREW PUNCTUATION GERSHAYIM:]
36 [:name = RIGHT SINGLE QUOTATION MARK:] [:name = HYPHENATION POINT:]];
38 $MidNum = [[:LineBreak = Infix_Numeric:] - [:name = FULL STOP:]];
39 $Numeric = [:LineBreak = Numeric:];
45 # Character Class Definitions.
46 # The names are those from TR29.
50 $Control = [[[:Zl:] [:Zp:] [:Cc:] [:Cf:]] - $TheZWSP];
51 $Extend = [[:Grapheme_Extend = TRUE:]];
56 ####################################################################################
58 # Word Break Rules. Definitions and Rules specific to word break begin Here.
60 ####################################################################################
62 $Format = [[:Cf:] - $TheZWSP];
66 # Rule 3: Treat a grapheme cluster as if it were a single character.
67 # Hangul Syllables are easier to deal with here than they are in Grapheme Clusters
68 # because we don't need to find the boundaries between adjacent syllables -
69 # they won't be word boundaries.
74 # "Extended" definitions. Grapheme Cluster + Format Chars, treated like the base char.
76 $ALetterEx = $ALetter $Extend*;
77 $NumericEx = $Numeric $Extend*;
78 $MidNumEx = $MidNum $Extend*;
79 $MidLetterEx = $MidLetter $Extend*;
80 $KatakanaEx = $Katakana $Extend*;
81 $IdeographicEx= $Ideographic $Extend*;
82 $HangulEx = $Hangul $Extend*;
83 $FormatEx = $Format $Extend*;
87 # Numbers. Rules 8, 11, 12 form the TR.
89 $NumberSequence = $NumericEx ($FormatEx* $MidNumEx? $FormatEx* $NumericEx)*;
90 $NumberSequence {100};
93 # Words. Alpha-numerics. Rule 5, 6, 7, 9, 10
94 # - must include at least one letter.
95 # - may include both letters and numbers.
96 # - may include MideLetter, MidNumber punctuation.
98 $LetterSequence = $ALetterEx ($FormatEx* $MidLetterEx? $FormatEx* $ALetterEx)*; # rules #6, #7
99 ($NumberSequence $FormatEx*)? $LetterSequence ($FormatEx* ($NumberSequence | $LetterSequence))* {200};
101 # Punctuations by themselves
102 [[:P:][:S:]-[:name = FULL STOP:]]*;
103 [[:name = FULL STOP:]]*;
106 # Do not break between Katakana. Rule #13.
108 $KatakanaEx ($FormatEx* $KatakanaEx)* {300};
109 [:Hiragana:] $Extend* {300};
112 # Ideographic Characters. Stand by themselves as words.
113 # Separated from the "Everything Else" rule, below, only so that they
114 # can be tagged with a return value. TODO: is this what we want?
116 $IdeographicEx ($FormatEx* $IdeographicEx)* {400};
117 $HangulEx ($FormatEx* $HangulEx)* {400};
120 # Everything Else, with no tag.
121 # Non-Control chars combine with $Extend (combining) chars.
122 # Controls are do not.
124 [^$Control [:Ideographic:]] $Extend*;
128 # Reverse Rules. Back up over any of the chars that can group together.
129 # (Reverse rules do not need to be exact; they can back up too far,
130 # but must back up at least enough, and must stop on a boundary.)
133 # NonStarters are the set of all characters that can appear at the 2nd - nth position of
134 # a word. (They may also be the first.) The reverse rule skips over these, until it
135 # reaches something that can only be the start (and probably only) char in a "word".
136 # A space or punctuation meets the test.
138 $NonStarters = [$Numeric $ALetter $Katakana $Ideographic $Hangul [:P:] [:S:] $MidLetter $MidNum $Extend $Format];
141 ! ($NonStarters* | \n \r) .;