| blob | 57aec04a15cfd2fc93dd75468841076a7042fc97 |
1 This is a dump from Google's source control system of the change
2 that removed UCS-2 support from RE2. As the explanation below
3 says, UCS-2 mode is fundamentally at odds with things like ^ and $,
4 so it never really worked very well. But if you are interested in using
5 it without those operators, it did work for that. It assumed that the
6 UCS-2 data was in the native host byte order.
8 If you are interested in adding UCS-2 mode back, this patch might
9 be a good starting point.
12 Change 12780686 by rsc@rsc-re2 on 2009/09/16 15:30:15
14 Retire UCS-2 mode.
16 I added it as an experiment for V8, but it
17 requires 2-byte lookahead to do completely,
18 and RE2 has 1-byte lookahead (enough for UTF-8)
19 as a fairly deep fundamental assumption,
20 so it did not support ^ or $.
22 ==== re2/bitstate.cc#2 - re2/bitstate.cc#3 ====
23 re2/bitstate.cc#2:314,321 - re2/bitstate.cc#3:314,319
24 cap_[0] = p;
25 if (TrySearch(prog_->start(), p)) // Match must be leftmost; done.
26 return true;
27 - if (prog_->flags() & Regexp::UCS2)
28 - p++;
29 }
30 return false;
31 }
32 ==== re2/compile.cc#17 - re2/compile.cc#18 ====
33 re2/compile.cc#17:95,101 - re2/compile.cc#18:95,100
34 // Input encodings.
35 enum Encoding {
36 kEncodingUTF8 = 1, // UTF-8 (0-10FFFF)
37 - kEncodingUCS2, // UCS-2 (0-FFFF), native byte order
38 kEncodingLatin1, // Latin1 (0-FF)
39 };
41 re2/compile.cc#17:168,176 - re2/compile.cc#18:167,172
42 void AddRuneRangeLatin1(Rune lo, Rune hi, bool foldcase);
43 void AddRuneRangeUTF8(Rune lo, Rune hi, bool foldcase);
44 void Add_80_10ffff();
45 - void AddRuneRangeUCS2(Rune lo, Rune hi, bool foldcase);
46 - void AddUCS2Pair(uint8 lo1, uint8 hi1, bool fold1,
47 - uint8 lo2, uint8 hi2, bool fold2);
49 // New suffix that matches the byte range lo-hi, then goes to next.
50 Inst* RuneByteSuffix(uint8 lo, uint8 hi, bool foldcase, Inst* next);
51 re2/compile.cc#17:475,481 - re2/compile.cc#18:471,477
53 // Converts rune range lo-hi into a fragment that recognizes
54 // the bytes that would make up those runes in the current
55 - // encoding (Latin 1, UTF-8, or UCS-2).
56 + // encoding (Latin 1 or UTF-8).
57 // This lets the machine work byte-by-byte even when
58 // using multibyte encodings.
60 re2/compile.cc#17:488,496 - re2/compile.cc#18:484,489
61 case kEncodingLatin1:
62 AddRuneRangeLatin1(lo, hi, foldcase);
63 break;
64 - case kEncodingUCS2:
65 - AddRuneRangeUCS2(lo, hi, foldcase);
66 - break;
67 }
68 }
70 re2/compile.cc#17:503,581 - re2/compile.cc#18:496,501
71 AddSuffix(RuneByteSuffix(lo, hi, foldcase, NULL));
72 }
74 - // Test whether 16-bit values are big or little endian.
75 - static bool BigEndian() {
76 - union {
77 - char byte[2];
78 - int16 endian;
79 - } u;
80 -
81 - u.byte[0] = 1;
82 - u.byte[1] = 2;
83 - return u.endian == 0x0102;
84 - }
85 -
86 - void Compiler::AddUCS2Pair(uint8 lo1, uint8 hi1, bool fold1,
87 - uint8 lo2, uint8 hi2, bool fold2) {
88 - Inst* ip;
89 - if (reversed_) {
90 - ip = RuneByteSuffix(lo1, hi1, fold1, NULL);
91 - ip = RuneByteSuffix(lo2, hi2, fold2, ip);
92 - } else {
93 - ip = RuneByteSuffix(lo2, hi2, fold2, NULL);
94 - ip = RuneByteSuffix(lo1, hi1, fold1, ip);
95 - }
96 - AddSuffix(ip);
97 - }
98 -
99 - void Compiler::AddRuneRangeUCS2(Rune lo, Rune hi, bool foldcase) {
100 - if (lo > hi || lo > 0xFFFF)
101 - return;
102 - if (hi > 0xFFFF)
103 - hi = 0xFFFF;
104 -
105 - // We'll assemble a pattern assuming big endian.
106 - // If the machine isn't, tell Cat to reverse its arguments.
107 - bool oldreversed = reversed_;
108 - if (!BigEndian()) {
109 - reversed_ = !oldreversed;
110 - }
111 -
112 - // Split into bytes.
113 - int lo1 = lo >> 8;
114 - int lo2 = lo & 0xFF;
115 - int hi1 = hi >> 8;
116 - int hi2 = hi & 0xFF;
117 -
118 - if (lo1 == hi1) {
119 - // Easy case: high bits are same in both.
120 - // Only do ASCII case folding on the second byte if the top byte is 00.
121 - AddUCS2Pair(lo1, lo1, false, lo2, hi2, lo1==0 && foldcase);
122 - } else {
123 - // Harder case: different second byte ranges depending on first byte.
124 -
125 - // Initial fragment.
126 - if (lo2 > 0) {
127 - AddUCS2Pair(lo1, lo1, false, lo2, 0xFF, lo1==0 && foldcase);
128 - lo1++;
129 - }
130 -
131 - // Trailing fragment.
132 - if (hi2 < 0xFF) {
133 - AddUCS2Pair(hi1, hi1, false, 0, hi2, false);
134 - hi1--;
135 - }
136 -
137 - // Inner ranges.
138 - if (lo1 <= hi1) {
139 - AddUCS2Pair(lo1, hi1, false, 0, 0xFF, false);
140 - }
141 - }
142 -
143 - // Restore reverse setting.
144 - reversed_ = oldreversed;
145 - }
146 -
147 // Table describing how to make a UTF-8 matching machine
148 // for the rune range 80-10FFFF (Runeself-Runemax).
149 // This range happens frequently enough (for example /./ and /[^a-z]/)
150 re2/compile.cc#17:707,716 - re2/compile.cc#18:627,634
152 Frag Compiler::Literal(Rune r, bool foldcase) {
153 switch (encoding_) {
154 - default: // UCS-2 or something new
155 - BeginRange();
156 - AddRuneRange(r, r, foldcase);
157 - return EndRange();
158 + default:
159 + return kNullFrag;
161 case kEncodingLatin1:
162 return ByteRange(r, r, foldcase);
163 re2/compile.cc#17:927,934 - re2/compile.cc#18:845,850
165 if (re->parse_flags() & Regexp::Latin1)
166 c.encoding_ = kEncodingLatin1;
167 - else if (re->parse_flags() & Regexp::UCS2)
168 - c.encoding_ = kEncodingUCS2;
169 c.reversed_ = reversed;
170 if (max_mem <= 0) {
171 c.max_inst_ = 100000; // more than enough
172 re2/compile.cc#17:983,993 - re2/compile.cc#18:899,905
173 c.prog_->set_start_unanchored(c.prog_->start());
174 } else {
175 Frag dot;
176 - if (c.encoding_ == kEncodingUCS2) {
177 - dot = c.Cat(c.ByteRange(0x00, 0xFF, false), c.ByteRange(0x00, 0xFF, false));
178 - } else {
179 - dot = c.ByteRange(0x00, 0xFF, false);
180 - }
181 + dot = c.ByteRange(0x00, 0xFF, false);
182 Frag dotloop = c.Star(dot, true);
183 Frag unanchored = c.Cat(dotloop, all);
184 c.prog_->set_start_unanchored(unanchored.begin);
185 ==== re2/nfa.cc#8 - re2/nfa.cc#9 ====
186 re2/nfa.cc#8:426,432 - re2/nfa.cc#9:426,431
187 const char* bp = context.begin();
188 int c = -1;
189 int wasword = 0;
190 - bool ucs2 = prog_->flags() & Regexp::UCS2;
192 if (text.begin() > context.begin()) {
193 c = text.begin()[-1] & 0xFF;
194 re2/nfa.cc#8:492,498 - re2/nfa.cc#9:491,497
195 // If there's a required first byte for an unanchored search
196 // and we're not in the middle of any possible matches,
197 // use memchr to search for the byte quickly.
198 - if (!ucs2 && !anchored && first_byte_ >= 0 && runq->size() == 0 &&
199 + if (!anchored && first_byte_ >= 0 && runq->size() == 0 &&
200 p < text.end() && (p[0] & 0xFF) != first_byte_) {
201 p = reinterpret_cast<const char*>(memchr(p, first_byte_,
202 text.end() - p));
203 re2/nfa.cc#8:505,526 - re2/nfa.cc#9:504,514
204 flag = Prog::EmptyFlags(context, p);
205 }
207 - // In UCS-2 mode, if we need to start a new thread,
208 - // make sure to do it on an even boundary.
209 - if(ucs2 && runq->size() == 0 &&
210 - (p - context.begin()) % 2 && p < text.end()) {
211 - p++;
212 - flag = Prog::EmptyFlags(context, p);
213 - }
214 -
215 // Steal match storage (cleared but unused as of yet)
216 // temporarily to hold match boundaries for new thread.
217 - // In UCS-2 mode, only start the thread on a 2-byte boundary.
218 - if(!ucs2 || (p - context.begin()) % 2 == 0) {
219 - match_[0] = p;
220 - AddToThreadq(runq, start_, flag, p, match_);
221 - match_[0] = NULL;
222 - }
223 + match_[0] = p;
224 + AddToThreadq(runq, start_, flag, p, match_);
225 + match_[0] = NULL;
226 }
228 // If all the threads have died, stop early.
229 ==== re2/parse.cc#22 - re2/parse.cc#23 ====
230 re2/parse.cc#22:160,167 - re2/parse.cc#23:160,165
231 status_(status), stacktop_(NULL), ncap_(0) {
232 if (flags_ & Latin1)
233 rune_max_ = 0xFF;
234 - else if (flags & UCS2)
235 - rune_max_ = 0xFFFF;
236 else
237 rune_max_ = Runemax;
238 }
239 re2/parse.cc#22:365,387 - re2/parse.cc#23:363,374
240 bool Regexp::ParseState::PushCarat() {
241 if (flags_ & OneLine) {
242 return PushSimpleOp(kRegexpBeginText);
243 - } else {
244 - if (flags_ & UCS2) {
245 - status_->set_code(kRegexpUnsupported);
246 - status_->set_error_arg("multiline ^ in UCS-2 mode");
247 - return false;
248 - }
249 - return PushSimpleOp(kRegexpBeginLine);
250 }
251 + return PushSimpleOp(kRegexpBeginLine);
252 }
254 // Pushes a \b or \B onto the stack.
255 bool Regexp::ParseState::PushWordBoundary(bool word) {
256 - if (flags_ & UCS2) {
257 - status_->set_code(kRegexpUnsupported);
258 - status_->set_error_arg("\\b or \\B in UCS-2 mode");
259 - return false;
260 - }
261 if (word)
262 return PushSimpleOp(kRegexpWordBoundary);
263 return PushSimpleOp(kRegexpNoWordBoundary);
264 re2/parse.cc#22:397,407 - re2/parse.cc#23:384,389
265 bool ret = PushSimpleOp(kRegexpEndText);
266 flags_ = oflags;
267 return ret;
268 - }
269 - if (flags_ & UCS2) {
270 - status_->set_code(kRegexpUnsupported);
271 - status_->set_error_arg("multiline $ in UCS-2 mode");
272 - return false;
273 }
274 return PushSimpleOp(kRegexpEndLine);
275 }
276 ==== re2/re2.cc#34 - re2/re2.cc#35 ====
277 re2/re2.cc#34:79,86 - re2/re2.cc#35:79,84
278 return RE2::ErrorBadUTF8;
279 case re2::kRegexpBadNamedCapture:
280 return RE2::ErrorBadNamedCapture;
281 - case re2::kRegexpUnsupported:
282 - return RE2::ErrorUnsupported;
283 }
284 return RE2::ErrorInternal;
285 }
286 re2/re2.cc#34:122,130 - re2/re2.cc#35:120,125
287 break;
288 case RE2::Options::EncodingLatin1:
289 flags |= Regexp::Latin1;
290 - break;
291 - case RE2::Options::EncodingUCS2:
292 - flags |= Regexp::UCS2;
293 break;
294 }
296 ==== re2/re2.h#36 - re2/re2.h#37 ====
297 re2/re2.h#36:246,252 - re2/re2.h#37:246,251
298 ErrorBadUTF8, // invalid UTF-8 in regexp
299 ErrorBadNamedCapture, // bad named capture group
300 ErrorPatternTooLarge, // pattern too large (compile failed)
301 - ErrorUnsupported, // unsupported feature (in UCS-2 mode)
302 };
304 // Predefined common options.
305 re2/re2.h#36:570,576 - re2/re2.h#37:569,574
307 enum Encoding {
308 EncodingUTF8 = 1,
309 - EncodingUCS2, // 16-bit Unicode 0-FFFF only
310 EncodingLatin1
311 };
313 ==== re2/regexp.cc#15 - re2/regexp.cc#16 ====
314 re2/regexp.cc#15:324,333 - re2/regexp.cc#16:324,329
315 // the regexp that remains after the prefix. The prefix might
316 // be ASCII case-insensitive.
317 bool Regexp::RequiredPrefix(string *prefix, bool *foldcase, Regexp** suffix) {
318 - // Don't even bother for UCS-2; it's time to throw that code away.
319 - if (parse_flags_ & UCS2)
320 - return false;
321 -
322 // No need for a walker: the regexp must be of the form
323 // 1. some number of ^ anchors
324 // 2. a literal char or string
325 ==== re2/regexp.h#20 - re2/regexp.h#21 ====
326 re2/regexp.h#20:187,193 - re2/regexp.h#21:187,192
327 kRegexpBadPerlOp, // bad perl operator
328 kRegexpBadUTF8, // invalid UTF-8 in regexp
329 kRegexpBadNamedCapture, // bad named capture
330 - kRegexpUnsupported, // unsupported operator
331 };
333 // Error status for certain operations.
334 re2/regexp.h#20:307,316 - re2/regexp.h#21:306,314
335 // \Q and \E to disable/enable metacharacters
336 // (?P<name>expr) for named captures
337 // \C to match any single byte
338 - UCS2 = 1<<10, // Text is in UCS-2, regexp is in UTF-8.
339 - UnicodeGroups = 1<<11, // Allow \p{Han} for Unicode Han group
340 + UnicodeGroups = 1<<10, // Allow \p{Han} for Unicode Han group
341 // and \P{Han} for its negation.
342 - NeverNL = 1<<12, // Never match NL, even if the regexp mentions
343 + NeverNL = 1<<11, // Never match NL, even if the regexp mentions
344 // it explicitly.
346 // As close to Perl as we can get.
347 ==== re2/testing/backtrack.cc#4 - re2/testing/backtrack.cc#5 ====
348 re2/testing/backtrack.cc#4:134,141 - re2/testing/backtrack.cc#5:134,139
349 cap_[0] = p;
350 if (Visit(prog_->start(), p)) // Match must be leftmost; done.
351 return true;
352 - if (prog_->flags() & Regexp::UCS2)
353 - p++;
354 }
355 return false;
356 }
357 ==== re2/testing/tester.cc#12 - re2/testing/tester.cc#13 ====
358 re2/testing/tester.cc#12:144,154 - re2/testing/tester.cc#13:144,152
359 static ParseMode parse_modes[] = {
360 { single_line, "single-line" },
361 { single_line|Regexp::Latin1, "single-line, latin1" },
362 - { single_line|Regexp::UCS2, "single-line, ucs2" },
363 { multi_line, "multiline" },
364 { multi_line|Regexp::NonGreedy, "multiline, nongreedy" },
365 { multi_line|Regexp::Latin1, "multiline, latin1" },
366 - { multi_line|Regexp::UCS2, "multiline, ucs2" },
367 };
369 static string FormatMode(Regexp::ParseFlags flags) {
370 re2/testing/tester.cc#12:179,189 - re2/testing/tester.cc#13:177,185
371 RegexpStatus status;
372 regexp_ = Regexp::Parse(regexp_str, flags, &status);
373 if (regexp_ == NULL) {
374 - if (status.code() != kRegexpUnsupported) {
375 - LOG(INFO) << "Cannot parse: " << CEscape(regexp_str_)
376 - << " mode: " << FormatMode(flags);
377 - error_ = true;
378 - }
379 + LOG(INFO) << "Cannot parse: " << CEscape(regexp_str_)
380 + << " mode: " << FormatMode(flags);
381 + error_ = true;
382 return;
383 }
384 prog_ = regexp_->CompileToProg(0);
385 re2/testing/tester.cc#12:230,237 - re2/testing/tester.cc#13:226,231
386 RE2::Options options;
387 if (flags & Regexp::Latin1)
388 options.set_encoding(RE2::Options::EncodingLatin1);
389 - else if (flags & Regexp::UCS2)
390 - options.set_encoding(RE2::Options::EncodingUCS2);
391 if (kind_ == Prog::kLongestMatch)
392 options.set_longest_match(true);
393 re2_ = new RE2(re, options);
394 re2/testing/tester.cc#12:281,379 - re2/testing/tester.cc#13:275,280
395 delete re2_;
396 }
398 - // Converts UTF-8 string in text into UCS-2 string in new_text.
399 - static bool ConvertUTF8ToUCS2(const StringPiece& text, StringPiece* new_text) {
400 - const char* p = text.begin();
401 - const char* ep = text.end();
402 - uint16* q = new uint16[ep - p];
403 - uint16* q0 = q;
404 -
405 - int n;
406 - Rune r;
407 - for (; p < ep; p += n) {
408 - if (!fullrune(p, ep - p)) {
409 - delete[] q0;
410 - return false;
411 - }
412 - n = chartorune(&r, p);
413 - if (r > 0xFFFF) {
414 - delete[] q0;
415 - return false;
416 - }
417 - *q++ = r;
418 - }
419 - *new_text = StringPiece(reinterpret_cast<char*>(q0), 2*(q - q0));
420 - return true;
421 - }
422 -
423 - // Rewrites *sp from being a pointer into text8 (UTF-8)
424 - // to being a pointer into text16 (equivalent text but in UCS-2).
425 - static void AdjustUTF8ToUCS2(const StringPiece& text8, const StringPiece& text16,
426 - StringPiece *sp) {
427 - if (sp->begin() == NULL && text8.begin() != NULL)
428 - return;
429 -
430 - int nrune = 0;
431 - int n;
432 - Rune r;
433 - const char* p = text8.begin();
434 - const char* ep = text8.end();
435 - const char* spbegin = NULL;
436 - const char* spend = NULL;
437 - for (;;) {
438 - if (p == sp->begin())
439 - spbegin = text16.begin() + sizeof(uint16)*nrune;
440 - if (p == sp->end())
441 - spend = text16.begin() + sizeof(uint16)*nrune;
442 - if (p >= ep)
443 - break;
444 - n = chartorune(&r, p);
445 - p += n;
446 - nrune++;
447 - }
448 - if (spbegin == NULL || spend == NULL) {
449 - LOG(FATAL) << "Error in AdjustUTF8ToUCS2 "
450 - << CEscape(text8) << " "
451 - << (int)(sp->begin() - text8.begin()) << " "
452 - << (int)(sp->end() - text8.begin());
453 - }
454 - *sp = StringPiece(spbegin, spend - spbegin);
455 - }
456 -
457 - // Rewrites *sp from begin a pointer into text16 (UCS-2)
458 - // to being a pointer into text8 (equivalent text but in UTF-8).
459 - static void AdjustUCS2ToUTF8(const StringPiece& text16, const StringPiece& text8,
460 - StringPiece* sp) {
461 - if (sp->begin() == NULL)
462 - return;
463 -
464 - int nrune = 0;
465 - int n;
466 - Rune r;
467 - const char* p = text8.begin();
468 - const char* ep = text8.end();
469 - const char* spbegin = NULL;
470 - const char* spend = NULL;
471 - for (;;) {
472 - if (nrune == (sp->begin() - text16.begin())/2)
473 - spbegin = p;
474 - if (nrune == (sp->end() - text16.begin())/2)
475 - spend = p;
476 - if (p >= ep)
477 - break;
478 - n = chartorune(&r, p);
479 - p += n;
480 - nrune++;
481 - }
482 - if (text8.begin() != NULL && (spbegin == NULL || spend == NULL)) {
483 - LOG(FATAL) << "Error in AdjustUCS2ToUTF8 "
484 - << CEscape(text16) << " "
485 - << (int)(sp->begin() - text16.begin()) << " "
486 - << (int)(sp->end() - text16.begin());
487 - }
488 - *sp = StringPiece(spbegin, spend - spbegin);
489 - }
490 -
491 // Runs a single search using the named engine type.
492 // This interface hides all the irregularities of the various
493 // engine interfaces from the rest of this file.
494 re2/testing/tester.cc#12:393,411 - re2/testing/tester.cc#13:294,300
496 StringPiece text = orig_text;
497 StringPiece context = orig_context;
498 - bool ucs2 = false;
500 - if ((flags() & Regexp::UCS2) && type != kEnginePCRE) {
501 - if (!ConvertUTF8ToUCS2(orig_context, &context)) {
502 - result->skipped = true;
503 - return;
504 - }
505 -
506 - // Rewrite context to refer to new text.
507 - AdjustUTF8ToUCS2(orig_context, context, &text);
508 - ucs2 = true;
509 - }
510 -
511 switch (type) {
512 default:
513 LOG(FATAL) << "Bad RunSearch type: " << (int)type;
514 re2/testing/tester.cc#12:557,577 - re2/testing/tester.cc#13:446,451
515 }
516 }
518 - // If we did UCS-2 matching, rewrite the matches to refer
519 - // to the original UTF-8 text.
520 - if (ucs2) {
521 - if (result->matched) {
522 - if (result->have_submatch0) {
523 - AdjustUCS2ToUTF8(context, orig_context, &result->submatch[0]);
524 - } else if (result->have_submatch) {
525 - for (int i = 0; i < nsubmatch; i++) {
526 - AdjustUCS2ToUTF8(context, orig_context, &result->submatch[i]);
527 - }
528 - }
529 - }
530 - delete[] context.begin();
531 - }
532 -
533 if (!result->matched)
534 memset(result->submatch, 0, sizeof result->submatch);
535 }
536 re2/testing/tester.cc#12:596,617 - re2/testing/tester.cc#13:470,475
537 return true;
538 }
540 - // Check whether text uses only Unicode points <= 0xFFFF
541 - // (in the BMP).
542 - static bool IsBMP(const StringPiece& text) {
543 - const char* p = text.begin();
544 - const char* ep = text.end();
545 - while (p < ep) {
546 - if (!fullrune(p, ep - p))
547 - return false;
548 - Rune r;
549 - p += chartorune(&r, p);
550 - if (r > 0xFFFF)
551 - return false;
552 - }
553 - return true;
554 - }
555 -
556 // Runs a single test.
557 bool TestInstance::RunCase(const StringPiece& text, const StringPiece& context,
558 Prog::Anchor anchor) {
559 re2/testing/tester.cc#12:619,625 - re2/testing/tester.cc#13:477,483
560 Result correct;
561 RunSearch(kEngineBacktrack, text, context, anchor, &correct);
562 if (correct.skipped) {
563 - if (regexp_ == NULL || !IsBMP(context)) // okay to skip in UCS-2 mode
564 + if (regexp_ == NULL)
565 return true;
566 LOG(ERROR) << "Skipped backtracking! " << CEscape(regexp_str_)
567 << " " << FormatMode(flags_);