2 * Copyright (C) 2002 Manuel Novoa III
3 * Copyright (C) 2000-2005 Erik Andersen <andersen@uclibc.org>
5 * Licensed under the LGPL v2.1, see the file COPYING.LIB in this tarball.
9 * Initial test implementation of strcoll, strxfrm, wcscoll, and wcsxfrm.
10 * The code needs to be cleaned up a good bit, but I'd like to see people
22 #ifdef __UCLIBC_HAS_LOCALE__
23 #if defined(L_strxfrm) || defined(L_strxfrm_l) || defined(L_wcsxfrm) || defined(L_wcsxfrm_l)
27 #error WANT_WIDE should be defined for L_strxfrm
30 #error L_wcsxfrm already defined for L_strxfrm
32 #endif /* L_strxfrm */
34 #if defined(L_strxfrm) || defined(L_strxfrm_l)
36 #define wcscoll strcoll
37 #define wcscoll_l strcoll_l
38 #define wcsxfrm strxfrm
39 #define wcsxfrm_l strxfrm_l
49 #endif /* defined(L_strxfrm) || defined(L_strxfrm_l) */
51 #if defined(__UCLIBC_HAS_XLOCALE__) && !defined(__UCLIBC_DO_XLOCALE)
54 int wcscoll (const Wchar
*s0
, const Wchar
*s1
)
56 return wcscoll_l(s0
, s1
, __UCLIBC_CURLOCALE
);
58 libc_hidden_def(wcscoll
)
61 size_t wcsxfrm(Wchar
*__restrict ws1
, const Wchar
*__restrict ws2
, size_t n
)
63 return wcsxfrm_l(ws1
, ws2
, n
, __UCLIBC_CURLOCALE
);
66 #else /* defined(__UCLIBC_HAS_XLOCALE__) && !defined(__UCLIBC_DO_XLOCALE) */
70 #define CUR_COLLATE (&__UCLIBC_CURLOCALE->collate)
72 #define CUR_COLLATE (& __LOCALE_PTR->collate)
79 const Wchar
*eob
; /* end of backward */
82 __uwchar_t ui_weight
; /* undefined or invalid */
87 /* should be wchar_t. if wchar < 0 do EILSEQ? */
89 __uwchar_t ci_pending
[MAX_PENDING
]; /* nul-terminated */
92 char *bbe
; /* end of back_buf (actual last... not 1 past end) */
93 char *bp
; /* ptr into backbuf, NULL if not in backward mode */
101 #define WEIGHT_MASK 0x3fffU
102 #define RULE_MASK 0xc000U
104 #define RULE_FORWARD (1 << 14)
105 #define RULE_POSITION (1 << 15)
107 #define UI_IDX (WEIGHT_MASK-6)
108 #define POSIT_IDX (WEIGHT_MASK-5)
109 #define RANGE_IDX (WEIGHT_MASK-4)
110 #define UNDEF_IDX (WEIGHT_MASK-3)
111 #define INVAL_IDX (WEIGHT_MASK-2)
112 #define DITTO_IDX (WEIGHT_MASK-1)
117 #define TRACE(X) printf X
119 #define TRACE(X) ((void)0)
122 static int lookup(wchar_t wc __LOCALE_PARAM
)
124 unsigned int sc
, n
, i0
, i1
;
126 if (((__uwchar_t
) wc
) > 0xffffU
) {
130 sc
= wc
& CUR_COLLATE
->ti_mask
;
131 wc
>>= CUR_COLLATE
->ti_shift
;
132 n
= wc
& CUR_COLLATE
->ii_mask
;
133 wc
>>= CUR_COLLATE
->ii_shift
;
135 i0
= CUR_COLLATE
->wcs2colidt_tbl
[wc
];
136 i0
<<= CUR_COLLATE
->ii_shift
;
137 i1
= CUR_COLLATE
->wcs2colidt_tbl
[CUR_COLLATE
->ii_len
+ i0
+ n
];
138 i1
<<= CUR_COLLATE
->ti_shift
;
139 return CUR_COLLATE
->wcs2colidt_tbl
[CUR_COLLATE
->ii_len
+ CUR_COLLATE
->ti_len
+ i1
+ sc
];
143 static void init_col_state(col_state_t
*cs
, const Wchar
*wcs
)
145 memset(cs
, 0, sizeof(col_state_t
));
147 cs
->bp
= cs
->back_buf
= cs
->ibb
;
149 cs
->bbe
= cs
->back_buf
+ (cs
->bb_size
-1);
152 static void next_weight(col_state_t
*cs
, int pass __LOCALE_PARAM
)
154 int r
, w
, ru
, ri
, popping_backup_stack
;
160 #else /* WANT_WIDE */
165 #endif /* WANT_WIDE */
171 TRACE(("ru_pushed = %d\n", ru
));
176 #ifdef __UCLIBC_MJN3_ONLY__
177 #warning should we walk pendings backwards?
179 if (cs
->cip
) { /* possible pending weight */
180 if ((r
= *(cs
->cip
++)) == 0) {
184 cs
->weightidx
= r
& WEIGHT_MASK
;
185 assert(cs
->weightidx
);
186 /* assert(cs->weightidx != WEIGHT_MASK); */
187 } else { /* get the next collation item from the string */
188 TRACE(("clearing popping flag\n"));
189 popping_backup_stack
= 0;
192 /* keep first pos as 0 for a sentinal */
193 if (*cs
->bp
) { /* pending backward chars */
195 popping_backup_stack
= 1;
196 TRACE(("setting popping flag\n"));
198 if (*cs
->bp
> 0) { /* singles pending */
200 if ((*cs
->bp
-= 1) == 0) {
203 } else { /* last was a multi */
207 } else if (!*cs
->s
) { /* not in backward mode and end of string */
217 cs
->colitem
= r
= lookup(*cs
->s __LOCALE_ARG
);
218 #else /* WANT_WIDE */
219 n
= n0
= __locale_mbrtowc_l(&WC
, cs
->s
, __LOCALE_PTR
);
225 cs
->colitem
= r
= lookup(WC __LOCALE_ARG
);
226 #endif /* WANT_WIDE */
228 TRACE((" r=%d WC=%#lx\n", r
, (unsigned long)(WC
)));
230 if (r
> CUR_COLLATE
->max_col_index
) { /* starting char for one or more sequences */
231 p
= CUR_COLLATE
->multistart_tbl
;
232 p
+= p
[r
-CUR_COLLATE
->max_col_index
-1];
237 if (!*p
) { /* found it */
239 TRACE((" found multi %d\n", n
));
243 /* the lookup check here is safe since we're assured that *p is a valid colidx */
244 if (!cs
->s
[n
] || (lookup(cs
->s
[n
] __LOCALE_ARG
) != *p
)) {
250 #else /* WANT_WIDE */
252 nx
= __locale_mbrtowc_l(&WC
, cs
->s
+ n
, __LOCALE_PTR
);
259 if (!cs
->s
[n
] || (lookup(WC __LOCALE_ARG
) != *p
)) {
264 n
+= nx
; /* Only gets here if cs->s[n] != 0, so nx is set. */
265 #endif /* WANT_WIDE */
268 } else if (r
== 0) { /* illegal, undefined, or part of a range */
269 if ((CUR_COLLATE
->range_count
)
270 #ifdef __UCLIBC_MJN3_ONLY__
271 #warning .. need to introduce range as a collating item?
273 && (((__uwchar_t
)(WC
- CUR_COLLATE
->range_low
)) <= CUR_COLLATE
->range_count
)
274 ) { /* part of a range */
275 /* Note: cs->colitem = 0 already. */
276 TRACE((" found range\n"));
277 ru
= CUR_COLLATE
->ruletable
[CUR_COLLATE
->range_rule_offset
*CUR_COLLATE
->MAX_WEIGHTS
+ pass
];
278 assert((ru
& WEIGHT_MASK
) != DITTO_IDX
);
279 if ((ru
& WEIGHT_MASK
) == WEIGHT_MASK
) {
280 ru
= (ru
& RULE_MASK
) | RANGE_IDX
;
281 cs
->weight
= CUR_COLLATE
->range_base_weight
+ (WC
- CUR_COLLATE
->range_low
);
284 } else if (((__uwchar_t
)(WC
)) <= 0x7fffffffUL
) { /* legal but undefined */
286 /* Note: cs->colitem = 0 already. */
287 ri
= CUR_COLLATE
->undefined_idx
;
288 assert(ri
!= 0); /* implicit undefined isn't supported */
290 TRACE((" found explicit UNDEFINED\n"));
291 #ifdef __UCLIBC_MJN3_ONLY__
292 #warning right now single weight locales do not support ..
294 if (CUR_COLLATE
->num_weights
== 1) {
295 TRACE((" single weight UNDEFINED\n"));
296 cs
->weightidx
= RANGE_IDX
;
302 ri
= CUR_COLLATE
->index2ruleidx
[ri
- 1];
303 ru
= CUR_COLLATE
->ruletable
[ri
* CUR_COLLATE
->MAX_WEIGHTS
+ pass
];
304 assert((ru
& WEIGHT_MASK
) != WEIGHT_MASK
); /* TODO: handle ".." */
305 if ((ru
& WEIGHT_MASK
) == DITTO_IDX
) {
306 cs
->colitem
= CUR_COLLATE
->undefined_idx
;
309 } else { /* illegal */
310 TRACE((" found illegal\n"));
312 /* We put all illegals in the same equiv class with maximal weight,
313 * and ignore them after the first pass. */
318 ru
= (RULE_FORWARD
| RANGE_IDX
);
319 cs
->weight
= 0xffffU
;
322 } else if (CUR_COLLATE
->num_weights
== 1) {
323 TRACE((" single weight\n"));
324 cs
->weightidx
= RANGE_IDX
;
325 cs
->weight
= cs
->colitem
;
329 TRACE((" normal\n"));
332 /* if we get here, it is a normal char either singlely weighted, undefined, or in a range */
334 ri
= CUR_COLLATE
->index2ruleidx
[cs
->colitem
- 1];
335 TRACE((" ri=%d ", ri
));
336 #ifdef __UCLIBC_MJN3_ONLY__
337 #warning make sure this is correct
340 TRACE(("NOT IN THIS LOCALE\n"));
343 ru
= CUR_COLLATE
->ruletable
[ri
* CUR_COLLATE
->MAX_WEIGHTS
+ pass
];
347 #ifdef __UCLIBC_MJN3_ONLY__
348 #warning ignoreables probably should not interrupt backwards processing, but this is wrong
350 /* if (!(ru & WEIGHT_MASK)) { */
351 /* TRACE(("IGNORE\n")); */
357 TRACE((" rule = %#x weight = %#x popping = %d s = %p eob = %p\n",
358 ru
& RULE_MASK
, ru
& WEIGHT_MASK
, popping_backup_stack
,
360 /* now we need to check if we're going backwards... */
362 if (!popping_backup_stack
) {
363 if (!(ru
& RULE_MASK
)) { /* backward */
364 TRACE(("backwards\n"));
365 assert(cs
->bp
<= cs
->bbe
);
366 if (cs
->bp
== cs
->bbe
) {
367 if (cs
->back_buf
== cs
->ibb
) { /* was using internal buffer */
368 cs
->bp
= malloc(cs
->bb_size
+ 128);
370 /* __set_errno(ENOMEM); */
371 #ifdef __UCLIBC_MJN3_ONLY__
372 #warning what to do here?
377 memcpy(cs
->bp
, cs
->back_buf
, cs
->bb_size
);
380 cs
->bp
= realloc(cs
->back_buf
, cs
->bb_size
+ 128);
382 /* __set_errno(ENOMEM); */
383 #ifdef __UCLIBC_MJN3_ONLY__
384 #warning what to do here?
391 cs
->bbe
= cs
->bp
+ (cs
->bbe
- cs
->back_buf
);
392 cs
->back_buf
= cs
->bp
;
396 if (n
==1) { /* single char */
397 if (*cs
->bp
&& (((unsigned char)(*cs
->bp
)) < CHAR_MAX
)) {
398 *cs
->bp
+= 1; /* increment last single's count */
399 } else { /* last was a multi, or just starting */
401 cs
->bp
= cs
->back_buf
;
403 assert(cs
->bp
< cs
->bbe
);
408 } else { /* multichar */
410 assert(cs
->bp
< cs
->bbe
);
417 /* end-of-string so start popping */
419 TRACE(("popping\n"));
421 } else if (*cs
->bp
) { /* was going backward but this element isn't */
422 /* discard current and use previous backward element */
425 TRACE(("popping\n"));
427 } else { /* was and still going forward */
428 TRACE(("forwards\n"));
429 if ((ru
& (RULE_POSITION
|WEIGHT_MASK
)) > RULE_POSITION
) {
430 assert(ru
& WEIGHT_MASK
);
432 cs
->weight
= cs
->position
;
433 #ifdef __UCLIBC_MJN3_ONLY__
436 cs
->position
= 0; /* reset to reduce size for strcoll? */
438 cs
->weightidx
= RANGE_IDX
;
442 } else { /* popping backwards stack */
443 TRACE(("popping (continued)\n"));
452 cs
->weightidx
= ru
& WEIGHT_MASK
;
453 cs
->rule
= ru
& RULE_MASK
;
456 #ifdef __UCLIBC_MJN3_ONLY__
457 #warning for pending we only want the weight... _not_ the rule
459 if (!cs
->weightidx
) { /* ignore */
464 assert(cs
->weightidx
);
467 if (((unsigned int)(cs
->weightidx
- UI_IDX
)) <= (INVAL_IDX
-UI_IDX
)) {
468 if (cs
->weightidx
== UI_IDX
) {
469 cs
->weight
= cs
->ui_weight
;
474 assert(cs
->weightidx
!= WEIGHT_MASK
);
475 if (cs
->weightidx
== DITTO_IDX
) { /* want the weight of the current collating item */
476 TRACE(("doing ditto\n"));
477 w
= CUR_COLLATE
->index2weight
[cs
->colitem
-1];
478 } else if (cs
->weightidx
<= CUR_COLLATE
->max_col_index
) { /* normal */
479 TRACE(("doing normal\n"));
480 w
= CUR_COLLATE
->index2weight
[cs
->weightidx
-1];
481 } else { /* a string */
482 TRACE(("doing string\n"));
483 assert(!(cs
->weightidx
& RULE_MASK
));
484 /* note: iso14561 allows null string here */
485 p
= CUR_COLLATE
->weightstr
+ (cs
->weightidx
- (CUR_COLLATE
->max_col_index
+ 2));
486 if (*p
& WEIGHT_MASK
) {
489 assert(r
< MAX_PENDING
);
490 cs
->ci_pending
[r
++] = *p
++;
491 } while (*p
& WEIGHT_MASK
);
492 cs
->cip
= cs
->ci_pending
;
502 int __XL_NPP(wcscoll
) (const Wchar
*s0
, const Wchar
*s1 __LOCALE_PARAM
)
507 if (!CUR_COLLATE
->num_weights
) { /* C locale */
509 return wcscmp(s0
, s1
);
511 return strcmp(s0
, s1
);
516 do { /* loop through the weights levels */
517 init_col_state(ws
, s0
);
518 init_col_state(ws
+1, s1
);
519 do { /* loop through the strings */
520 /* for each string, get the next weight */
521 next_weight(ws
, pass __LOCALE_ARG
);
522 next_weight(ws
+1, pass __LOCALE_ARG
);
523 TRACE(("w0=%lu w1=%lu\n",
524 (unsigned long) ws
[0].weight
,
525 (unsigned long) ws
[1].weight
));
527 if (ws
[0].weight
!= ws
[1].weight
) {
528 return ws
[0].weight
- ws
[1].weight
;
530 } while (ws
[0].weight
);
531 } while (++pass
< CUR_COLLATE
->num_weights
);
535 libc_hidden_def(__XL_NPP(wcscoll
))
539 size_t __XL_NPP(wcsxfrm
)(wchar_t *__restrict ws1
, const wchar_t *__restrict ws2
,
540 size_t n __LOCALE_PARAM
)
546 if (!CUR_COLLATE
->num_weights
) { /* C locale */
547 return __wcslcpy(ws1
, ws2
, n
);
550 #ifdef __UCLIBC_MJN3_ONLY__
551 #warning handle empty string as a special case
555 do { /* loop through the weights levels */
556 init_col_state(&cs
, ws2
);
557 do { /* loop through the string */
558 next_weight(&cs
, pass __LOCALE_ARG
);
559 TRACE(("weight=%lu (%#lx)\n", (unsigned long) cs
.weight
, (unsigned long) cs
.weight
));
561 ws1
[count
] = cs
.weight
+1;
564 TRACE(("--------------------------------------------\n"));
566 if (count
<= n
) { /* overwrite the trailing 0 end-of-pass marker */
569 TRACE(("-------------------- pass %d --------------------\n", pass
));
570 } while (++pass
< CUR_COLLATE
->num_weights
);
571 if (count
<= n
) { /* oops... change it back */
576 #if defined L_strxfrm_l || defined L_wcsxfrm_l
577 libc_hidden_def(__XL_NPP(wcsxfrm
))
580 #else /* WANT_WIDE */
582 static const unsigned long bound
[] = {
590 static unsigned char first
[] = {
591 0x0, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc
594 /* Use an extension of UTF-8 to store a 32 bit val in max 6 bytes. */
596 static size_t store(unsigned char *s
, size_t count
, size_t n
, __uwchar_t weight
)
602 if (weight
< bound
[i
]) {
605 } while (++i
< sizeof(bound
)/sizeof(bound
[0]));
612 s
[i
] = 0x80 | (weight
& 0x3f);
622 size_t __XL_NPP(strxfrm
)(char *__restrict ws1
, const char *__restrict ws2
, size_t n
629 if (!CUR_COLLATE
->num_weights
) { /* C locale */
630 return strlcpy(ws1
, ws2
, n
);
633 #ifdef __UCLIBC_MJN3_ONLY__
634 #warning handle empty string as a special case
637 inc
= count
= pass
= 0;
638 do { /* loop through the weights levels */
639 init_col_state(&cs
, ws2
);
640 do { /* loop through the string */
641 next_weight(&cs
, pass __LOCALE_ARG
);
642 TRACE(("weight=%lu (%#lx)\n", (unsigned long) cs
.weight
, (unsigned long) cs
.weight
));
643 inc
= store((unsigned char *)ws1
, count
, n
, cs
.weight
+ 1);
645 TRACE(("--------------------------------------------\n"));
647 /* overwrite the trailing 0 end-of-pass marker */
652 TRACE(("-------------------- pass %d --------------------\n", pass
));
653 } while (++pass
< CUR_COLLATE
->num_weights
);
654 if (count
<= n
) { /* oops... change it back */
660 libc_hidden_def(__XL_NPP(strxfrm
))
663 #endif /* WANT_WIDE */
665 #endif /* defined(__UCLIBC_HAS_XLOCALE__) && !defined(__UCLIBC_DO_XLOCALE) */
667 #endif /* defined(L_strxfrm) || defined(L_strxfrm_l) || defined(L_wcsxfrm) || defined(L_wcsxfrm_l) */
669 #endif /* __UCLIBC_HAS_LOCALE__ */
670 /**********************************************************************/