1 /* gutf8.c - Operations on UTF-8 strings.
3 * Copyright (C) 1999 Tom Tromey
4 * Copyright (C) 2000 Red Hat, Inc.
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 02111-1307, USA.
30 #ifdef G_PLATFORM_WIN32
37 #include "libcharset/libcharset.h"
41 #include "gstrfuncs.h"
42 #include "gtestutils.h"
47 #define UTF8_COMPUTE(Char, Mask, Len) \
53 else if ((Char & 0xe0) == 0xc0) \
58 else if ((Char & 0xf0) == 0xe0) \
63 else if ((Char & 0xf8) == 0xf0) \
68 else if ((Char & 0xfc) == 0xf8) \
73 else if ((Char & 0xfe) == 0xfc) \
81 #define UTF8_LENGTH(Char) \
82 ((Char) < 0x80 ? 1 : \
83 ((Char) < 0x800 ? 2 : \
84 ((Char) < 0x10000 ? 3 : \
85 ((Char) < 0x200000 ? 4 : \
86 ((Char) < 0x4000000 ? 5 : 6)))))
89 #define UTF8_GET(Result, Chars, Count, Mask, Len) \
90 (Result) = (Chars)[0] & (Mask); \
91 for ((Count) = 1; (Count) < (Len); ++(Count)) \
93 if (((Chars)[(Count)] & 0xc0) != 0x80) \
99 (Result) |= ((Chars)[(Count)] & 0x3f); \
103 * Check whether a Unicode (5.2) char is in a valid range.
105 * The first check comes from the Unicode guarantee to never encode
106 * a point above 0x0010ffff, since UTF-16 couldn't represent it.
108 * The second check covers surrogate pairs (category Cs).
110 * The last two checks cover "Noncharacter": defined as:
111 * "A code point that is permanently reserved for
112 * internal use, and that should never be interchanged. In
113 * Unicode 3.1, these consist of the values U+nFFFE and U+nFFFF
114 * (where n is from 0 to 10_16) and the values U+FDD0..U+FDEF."
116 * @param Char the character
118 #define UNICODE_VALID(Char) \
119 ((Char) < 0x110000 && \
120 (((Char) & 0xFFFFF800) != 0xD800) && \
121 ((Char) < 0xFDD0 || (Char) > 0xFDEF) && \
122 ((Char) & 0xFFFE) != 0xFFFE)
125 static const gchar utf8_skip_data
[256] = {
126 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
127 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
128 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
129 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
130 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
131 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
132 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
133 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,1,1
136 const gchar
* const g_utf8_skip
= utf8_skip_data
;
139 * g_utf8_find_prev_char:
140 * @str: pointer to the beginning of a UTF-8 encoded string
141 * @p: pointer to some position within @str
143 * Given a position @p with a UTF-8 encoded string @str, find the start
144 * of the previous UTF-8 character starting before @p. Returns %NULL if no
145 * UTF-8 characters are present in @str before @p.
147 * @p does not have to be at the beginning of a UTF-8 character. No check
148 * is made to see if the character found is actually valid other than
149 * it starts with an appropriate byte.
151 * Return value: a pointer to the found character or %NULL.
154 g_utf8_find_prev_char (const char *str
,
157 for (--p
; p
>= str
; --p
)
159 if ((*p
& 0xc0) != 0x80)
166 * g_utf8_find_next_char:
167 * @p: a pointer to a position within a UTF-8 encoded string
168 * @end: a pointer to the byte following the end of the string,
169 * or %NULL to indicate that the string is nul-terminated.
171 * Finds the start of the next UTF-8 character in the string after @p.
173 * @p does not have to be at the beginning of a UTF-8 character. No check
174 * is made to see if the character found is actually valid other than
175 * it starts with an appropriate byte.
177 * Return value: a pointer to the found character or %NULL
180 g_utf8_find_next_char (const gchar
*p
,
186 for (++p
; p
< end
&& (*p
& 0xc0) == 0x80; ++p
)
189 for (++p
; (*p
& 0xc0) == 0x80; ++p
)
192 return (p
== end
) ? NULL
: (gchar
*)p
;
197 * @p: a pointer to a position within a UTF-8 encoded string
199 * Finds the previous UTF-8 character in the string before @p.
201 * @p does not have to be at the beginning of a UTF-8 character. No check
202 * is made to see if the character found is actually valid other than
203 * it starts with an appropriate byte. If @p might be the first
204 * character of the string, you must use g_utf8_find_prev_char() instead.
206 * Return value: a pointer to the found character.
209 g_utf8_prev_char (const gchar
*p
)
214 if ((*p
& 0xc0) != 0x80)
221 * @p: pointer to the start of a UTF-8 encoded string
222 * @max: the maximum number of bytes to examine. If @max
223 * is less than 0, then the string is assumed to be
224 * nul-terminated. If @max is 0, @p will not be examined and
227 * Computes the length of the string in characters, not including
228 * the terminating nul character.
230 * Return value: the length of the string in characters
233 g_utf8_strlen (const gchar
*p
,
237 const gchar
*start
= p
;
238 g_return_val_if_fail (p
!= NULL
|| max
== 0, 0);
244 p
= g_utf8_next_char (p
);
253 p
= g_utf8_next_char (p
);
255 while (p
- start
< max
&& *p
)
258 p
= g_utf8_next_char (p
);
261 /* only do the last len increment if we got a complete
262 * char (don't count partial chars)
264 if (p
- start
<= max
)
273 * @p: a pointer to Unicode character encoded as UTF-8
275 * Converts a sequence of bytes encoded as UTF-8 to a Unicode character.
276 * If @p does not point to a valid UTF-8 encoded character, results are
277 * undefined. If you are not sure that the bytes are complete
278 * valid Unicode characters, you should use g_utf8_get_char_validated()
281 * Return value: the resulting character
284 g_utf8_get_char (const gchar
*p
)
286 int i
, mask
= 0, len
;
288 unsigned char c
= (unsigned char) *p
;
290 UTF8_COMPUTE (c
, mask
, len
);
293 UTF8_GET (result
, p
, i
, mask
, len
);
299 * g_utf8_offset_to_pointer:
300 * @str: a UTF-8 encoded string
301 * @offset: a character offset within @str
303 * Converts from an integer character offset to a pointer to a position
306 * Since 2.10, this function allows to pass a negative @offset to
307 * step backwards. It is usually worth stepping backwards from the end
308 * instead of forwards if @offset is in the last fourth of the string,
309 * since moving forward is about 3 times faster than moving backward.
312 * This function doesn't abort when reaching the end of @str. Therefore
313 * you should be sure that @offset is within string boundaries before
314 * calling that function. Call g_utf8_strlen() when unsure.
316 * This limitation exists as this function is called frequently during
317 * text rendering and therefore has to be as fast as possible.
320 * Return value: the resulting pointer
323 g_utf8_offset_to_pointer (const gchar
*str
,
326 const gchar
*s
= str
;
330 s
= g_utf8_next_char (s
);
335 /* This nice technique for fast backwards stepping
336 * through a UTF-8 string was dubbed "stutter stepping"
337 * by its inventor, Larry Ewing.
343 while ((*s
& 0xc0) == 0x80)
346 offset
+= g_utf8_pointer_to_offset (s
, s1
);
354 * g_utf8_pointer_to_offset:
355 * @str: a UTF-8 encoded string
356 * @pos: a pointer to a position within @str
358 * Converts from a pointer to position within a string to a integer
361 * Since 2.10, this function allows @pos to be before @str, and returns
362 * a negative offset in this case.
364 * Return value: the resulting character offset
367 g_utf8_pointer_to_offset (const gchar
*str
,
370 const gchar
*s
= str
;
374 offset
= - g_utf8_pointer_to_offset (pos
, str
);
378 s
= g_utf8_next_char (s
);
388 * @dest: buffer to fill with characters from @src
389 * @src: UTF-8 encoded string
390 * @n: character count
392 * Like the standard C strncpy() function, but
393 * copies a given number of characters instead of a given number of
394 * bytes. The @src string must be valid UTF-8 encoded text.
395 * (Use g_utf8_validate() on all text before trying to use UTF-8
396 * utility functions with it.)
398 * Return value: @dest
401 g_utf8_strncpy (gchar
*dest
,
405 const gchar
*s
= src
;
408 s
= g_utf8_next_char(s
);
411 strncpy(dest
, src
, s
- src
);
416 G_LOCK_DEFINE_STATIC (aliases
);
419 get_alias_hash (void)
421 static GHashTable
*alias_hash
= NULL
;
428 alias_hash
= g_hash_table_new (g_str_hash
, g_str_equal
);
430 aliases
= _g_locale_get_charset_aliases ();
431 while (*aliases
!= '\0')
433 const char *canonical
;
435 const char **alias_array
;
439 aliases
+= strlen (aliases
) + 1;
441 aliases
+= strlen (aliases
) + 1;
443 alias_array
= g_hash_table_lookup (alias_hash
, canonical
);
446 while (alias_array
[count
])
450 alias_array
= g_renew (const char *, alias_array
, count
+ 2);
451 alias_array
[count
] = alias
;
452 alias_array
[count
+ 1] = NULL
;
454 g_hash_table_insert (alias_hash
, (char *)canonical
, alias_array
);
463 /* As an abuse of the alias table, the following routines gets
464 * the charsets that are aliases for the canonical name.
466 G_GNUC_INTERNAL
const char **
467 _g_charset_get_aliases (const char *canonical_name
)
469 GHashTable
*alias_hash
= get_alias_hash ();
471 return g_hash_table_lookup (alias_hash
, canonical_name
);
475 g_utf8_get_charset_internal (const char *raw_data
,
478 const char *charset
= getenv("CHARSET");
480 if (charset
&& *charset
)
484 if (charset
&& strstr (charset
, "UTF-8"))
490 /* The libcharset code tries to be thread-safe without
491 * a lock, but has a memory leak and a missing memory
492 * barrier, so we lock for it
495 charset
= _g_locale_charset_unalias (raw_data
);
498 if (charset
&& *charset
)
502 if (charset
&& strstr (charset
, "UTF-8"))
508 /* Assume this for compatibility at present. */
514 typedef struct _GCharsetCache GCharsetCache
;
516 struct _GCharsetCache
{
523 charset_cache_free (gpointer data
)
525 GCharsetCache
*cache
= data
;
527 g_free (cache
->charset
);
533 * @charset: return location for character set name
535 * Obtains the character set for the <link linkend="setlocale">current
536 * locale</link>; you might use this character set as an argument to
537 * g_convert(), to convert from the current locale's encoding to some
538 * other encoding. (Frequently g_locale_to_utf8() and g_locale_from_utf8()
539 * are nice shortcuts, though.)
541 * On Windows the character set returned by this function is the
542 * so-called system default ANSI code-page. That is the character set
543 * used by the "narrow" versions of C library and Win32 functions that
544 * handle file names. It might be different from the character set
545 * used by the C library's current locale.
547 * The return value is %TRUE if the locale's encoding is UTF-8, in that
548 * case you can perhaps avoid calling g_convert().
550 * The string returned in @charset is not allocated, and should not be
553 * Return value: %TRUE if the returned charset is UTF-8
556 g_get_charset (G_CONST_RETURN
char **charset
)
558 static GStaticPrivate cache_private
= G_STATIC_PRIVATE_INIT
;
559 GCharsetCache
*cache
= g_static_private_get (&cache_private
);
564 cache
= g_new0 (GCharsetCache
, 1);
565 g_static_private_set (&cache_private
, cache
, charset_cache_free
);
568 raw
= _g_locale_charset_raw ();
570 if (!(cache
->raw
&& strcmp (cache
->raw
, raw
) == 0))
572 const gchar
*new_charset
;
575 g_free (cache
->charset
);
576 cache
->raw
= g_strdup (raw
);
577 cache
->is_utf8
= g_utf8_get_charset_internal (raw
, &new_charset
);
578 cache
->charset
= g_strdup (new_charset
);
582 *charset
= cache
->charset
;
584 return cache
->is_utf8
;
591 * @c: a Unicode character code
592 * @outbuf: output buffer, must have at least 6 bytes of space.
593 * If %NULL, the length will be computed and returned
594 * and nothing will be written to @outbuf.
596 * Converts a single character to UTF-8.
598 * Return value: number of bytes written
601 g_unichar_to_utf8 (gunichar c
,
604 /* If this gets modified, also update the copy in g_string_insert_unichar() */
619 else if (c
< 0x10000)
624 else if (c
< 0x200000)
629 else if (c
< 0x4000000)
642 for (i
= len
- 1; i
> 0; --i
)
644 outbuf
[i
] = (c
& 0x3f) | 0x80;
647 outbuf
[0] = c
| first
;
655 * @p: a nul-terminated UTF-8 encoded string
656 * @len: the maximum length of @p
657 * @c: a Unicode character
659 * Finds the leftmost occurrence of the given Unicode character
660 * in a UTF-8 encoded string, while limiting the search to @len bytes.
661 * If @len is -1, allow unbounded search.
663 * Return value: %NULL if the string does not contain the character,
664 * otherwise, a pointer to the start of the leftmost occurrence of
665 * the character in the string.
668 g_utf8_strchr (const char *p
,
674 gint charlen
= g_unichar_to_utf8 (c
, ch
);
677 return g_strstr_len (p
, len
, ch
);
683 * @p: a nul-terminated UTF-8 encoded string
684 * @len: the maximum length of @p
685 * @c: a Unicode character
687 * Find the rightmost occurrence of the given Unicode character
688 * in a UTF-8 encoded string, while limiting the search to @len bytes.
689 * If @len is -1, allow unbounded search.
691 * Return value: %NULL if the string does not contain the character,
692 * otherwise, a pointer to the start of the rightmost occurrence of the
693 * character in the string.
696 g_utf8_strrchr (const char *p
,
702 gint charlen
= g_unichar_to_utf8 (c
, ch
);
705 return g_strrstr_len (p
, len
, ch
);
709 /* Like g_utf8_get_char, but take a maximum length
710 * and return (gunichar)-2 on incomplete trailing character;
711 * also check for malformed or overlong sequences
712 * and return (gunichar)-1 in this case.
714 static inline gunichar
715 g_utf8_get_char_extended (const gchar
*p
,
720 gunichar wc
= (guchar
) *p
;
726 else if (G_UNLIKELY (wc
< 0xc0))
765 if (G_UNLIKELY (max_len
>= 0 && len
> max_len
))
767 for (i
= 1; i
< max_len
; i
++)
769 if ((((guchar
*)p
)[i
] & 0xc0) != 0x80)
775 for (i
= 1; i
< len
; ++i
)
777 gunichar ch
= ((guchar
*)p
)[i
];
779 if (G_UNLIKELY ((ch
& 0xc0) != 0x80))
791 if (G_UNLIKELY (wc
< min_code
))
798 * g_utf8_get_char_validated:
799 * @p: a pointer to Unicode character encoded as UTF-8
800 * @max_len: the maximum number of bytes to read, or -1, for no maximum or
801 * if @p is nul-terminated
803 * Convert a sequence of bytes encoded as UTF-8 to a Unicode character.
804 * This function checks for incomplete characters, for invalid characters
805 * such as characters that are out of the range of Unicode, and for
806 * overlong encodings of valid characters.
808 * Return value: the resulting character. If @p points to a partial
809 * sequence at the end of a string that could begin a valid
810 * character (or if @max_len is zero), returns (gunichar)-2;
811 * otherwise, if @p does not point to a valid UTF-8 encoded
812 * Unicode character, returns (gunichar)-1.
815 g_utf8_get_char_validated (const gchar
*p
,
823 result
= g_utf8_get_char_extended (p
, max_len
);
825 if (result
& 0x80000000)
827 else if (!UNICODE_VALID (result
))
834 * g_utf8_to_ucs4_fast:
835 * @str: a UTF-8 encoded string
836 * @len: the maximum length of @str to use, in bytes. If @len < 0,
837 * then the string is nul-terminated.
838 * @items_written: location to store the number of characters in the
841 * Convert a string from UTF-8 to a 32-bit fixed width
842 * representation as UCS-4, assuming valid UTF-8 input.
843 * This function is roughly twice as fast as g_utf8_to_ucs4()
844 * but does no error checking on the input.
846 * Return value: a pointer to a newly allocated UCS-4 string.
847 * This value must be freed with g_free().
850 g_utf8_to_ucs4_fast (const gchar
*str
,
852 glong
*items_written
)
859 g_return_val_if_fail (str
!= NULL
, NULL
);
867 p
= g_utf8_next_char (p
);
873 while (p
< str
+ len
&& *p
)
875 p
= g_utf8_next_char (p
);
880 result
= g_new (gunichar
, n_chars
+ 1);
883 for (i
=0; i
< n_chars
; i
++)
885 gunichar wc
= ((unsigned char *)p
)[0];
920 for (j
= 1; j
< charlen
; j
++)
923 wc
|= ((unsigned char *)p
)[j
] & 0x3f;
940 * @str: a UTF-8 encoded string
941 * @len: the maximum length of @str to use, in bytes. If @len < 0,
942 * then the string is nul-terminated.
943 * @items_read: location to store number of bytes read, or %NULL.
944 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
945 * returned in case @str contains a trailing partial
946 * character. If an error occurs then the index of the
947 * invalid input is stored here.
948 * @items_written: location to store number of characters written or %NULL.
949 * The value here stored does not include the trailing 0
951 * @error: location to store the error occuring, or %NULL to ignore
952 * errors. Any of the errors in #GConvertError other than
953 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
955 * Convert a string from UTF-8 to a 32-bit fixed width
956 * representation as UCS-4. A trailing 0 will be added to the
957 * string after the converted text.
959 * Return value: a pointer to a newly allocated UCS-4 string.
960 * This value must be freed with g_free(). If an
961 * error occurs, %NULL will be returned and
965 g_utf8_to_ucs4 (const gchar
*str
,
968 glong
*items_written
,
971 gunichar
*result
= NULL
;
977 while ((len
< 0 || str
+ len
- in
> 0) && *in
)
979 gunichar wc
= g_utf8_get_char_extended (in
, len
< 0 ? 6 : str
+ len
- in
);
982 if (wc
== (gunichar
)-2)
987 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_PARTIAL_INPUT
,
988 _("Partial character sequence at end of input"));
991 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
992 _("Invalid byte sequence in conversion input"));
999 in
= g_utf8_next_char (in
);
1002 result
= g_new (gunichar
, n_chars
+ 1);
1005 for (i
=0; i
< n_chars
; i
++)
1007 result
[i
] = g_utf8_get_char (in
);
1008 in
= g_utf8_next_char (in
);
1013 *items_written
= n_chars
;
1017 *items_read
= in
- str
;
1024 * @str: a UCS-4 encoded string
1025 * @len: the maximum length (number of characters) of @str to use.
1026 * If @len < 0, then the string is nul-terminated.
1027 * @items_read: location to store number of characters read, or %NULL.
1028 * @items_written: location to store number of bytes written or %NULL.
1029 * The value here stored does not include the trailing 0
1031 * @error: location to store the error occuring, or %NULL to ignore
1032 * errors. Any of the errors in #GConvertError other than
1033 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1035 * Convert a string from a 32-bit fixed width representation as UCS-4.
1036 * to UTF-8. The result will be terminated with a 0 byte.
1038 * Return value: a pointer to a newly allocated UTF-8 string.
1039 * This value must be freed with g_free(). If an
1040 * error occurs, %NULL will be returned and
1041 * @error set. In that case, @items_read will be
1042 * set to the position of the first invalid input
1046 g_ucs4_to_utf8 (const gunichar
*str
,
1049 glong
*items_written
,
1053 gchar
*result
= NULL
;
1058 for (i
= 0; len
< 0 || i
< len
; i
++)
1063 if (str
[i
] >= 0x80000000)
1065 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1066 _("Character out of range for UTF-8"));
1070 result_length
+= UTF8_LENGTH (str
[i
]);
1073 result
= g_malloc (result_length
+ 1);
1077 while (p
< result
+ result_length
)
1078 p
+= g_unichar_to_utf8 (str
[i
++], p
);
1083 *items_written
= p
- result
;
1092 #define SURROGATE_VALUE(h,l) (((h) - 0xd800) * 0x400 + (l) - 0xdc00 + 0x10000)
1096 * @str: a UTF-16 encoded string
1097 * @len: the maximum length (number of <type>gunichar2</type>) of @str to use.
1098 * If @len < 0, then the string is nul-terminated.
1099 * @items_read: location to store number of words read, or %NULL.
1100 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1101 * returned in case @str contains a trailing partial
1102 * character. If an error occurs then the index of the
1103 * invalid input is stored here.
1104 * @items_written: location to store number of bytes written, or %NULL.
1105 * The value stored here does not include the trailing
1107 * @error: location to store the error occuring, or %NULL to ignore
1108 * errors. Any of the errors in #GConvertError other than
1109 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1111 * Convert a string from UTF-16 to UTF-8. The result will be
1112 * terminated with a 0 byte.
1114 * Note that the input is expected to be already in native endianness,
1115 * an initial byte-order-mark character is not handled specially.
1116 * g_convert() can be used to convert a byte buffer of UTF-16 data of
1117 * ambiguous endianess.
1119 * Further note that this function does not validate the result
1120 * string; it may e.g. include embedded NUL characters. The only
1121 * validation done by this function is to ensure that the input can
1122 * be correctly interpreted as UTF-16, i.e. it doesn't contain
1123 * things unpaired surrogates.
1125 * Return value: a pointer to a newly allocated UTF-8 string.
1126 * This value must be freed with g_free(). If an
1127 * error occurs, %NULL will be returned and
1131 g_utf16_to_utf8 (const gunichar2
*str
,
1134 glong
*items_written
,
1137 /* This function and g_utf16_to_ucs4 are almost exactly identical - The lines that differ
1140 const gunichar2
*in
;
1142 gchar
*result
= NULL
;
1144 gunichar high_surrogate
;
1146 g_return_val_if_fail (str
!= NULL
, NULL
);
1151 while ((len
< 0 || in
- str
< len
) && *in
)
1156 if (c
>= 0xdc00 && c
< 0xe000) /* low surrogate */
1160 wc
= SURROGATE_VALUE (high_surrogate
, c
);
1165 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1166 _("Invalid sequence in conversion input"));
1174 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1175 _("Invalid sequence in conversion input"));
1179 if (c
>= 0xd800 && c
< 0xdc00) /* high surrogate */
1188 /********** DIFFERENT for UTF8/UCS4 **********/
1189 n_bytes
+= UTF8_LENGTH (wc
);
1195 if (high_surrogate
&& !items_read
)
1197 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_PARTIAL_INPUT
,
1198 _("Partial character sequence at end of input"));
1202 /* At this point, everything is valid, and we just need to convert
1204 /********** DIFFERENT for UTF8/UCS4 **********/
1205 result
= g_malloc (n_bytes
+ 1);
1210 while (out
< result
+ n_bytes
)
1215 if (c
>= 0xdc00 && c
< 0xe000) /* low surrogate */
1217 wc
= SURROGATE_VALUE (high_surrogate
, c
);
1220 else if (c
>= 0xd800 && c
< 0xdc00) /* high surrogate */
1228 /********** DIFFERENT for UTF8/UCS4 **********/
1229 out
+= g_unichar_to_utf8 (wc
, out
);
1235 /********** DIFFERENT for UTF8/UCS4 **********/
1239 /********** DIFFERENT for UTF8/UCS4 **********/
1240 *items_written
= out
- result
;
1244 *items_read
= in
- str
;
1251 * @str: a UTF-16 encoded string
1252 * @len: the maximum length (number of <type>gunichar2</type>) of @str to use.
1253 * If @len < 0, then the string is nul-terminated.
1254 * @items_read: location to store number of words read, or %NULL.
1255 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1256 * returned in case @str contains a trailing partial
1257 * character. If an error occurs then the index of the
1258 * invalid input is stored here.
1259 * @items_written: location to store number of characters written, or %NULL.
1260 * The value stored here does not include the trailing
1262 * @error: location to store the error occuring, or %NULL to ignore
1263 * errors. Any of the errors in #GConvertError other than
1264 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1266 * Convert a string from UTF-16 to UCS-4. The result will be
1269 * Return value: a pointer to a newly allocated UCS-4 string.
1270 * This value must be freed with g_free(). If an
1271 * error occurs, %NULL will be returned and
1275 g_utf16_to_ucs4 (const gunichar2
*str
,
1278 glong
*items_written
,
1281 const gunichar2
*in
;
1283 gchar
*result
= NULL
;
1285 gunichar high_surrogate
;
1287 g_return_val_if_fail (str
!= NULL
, NULL
);
1292 while ((len
< 0 || in
- str
< len
) && *in
)
1297 if (c
>= 0xdc00 && c
< 0xe000) /* low surrogate */
1301 wc
= SURROGATE_VALUE (high_surrogate
, c
);
1306 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1307 _("Invalid sequence in conversion input"));
1315 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1316 _("Invalid sequence in conversion input"));
1320 if (c
>= 0xd800 && c
< 0xdc00) /* high surrogate */
1329 /********** DIFFERENT for UTF8/UCS4 **********/
1330 n_bytes
+= sizeof (gunichar
);
1336 if (high_surrogate
&& !items_read
)
1338 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_PARTIAL_INPUT
,
1339 _("Partial character sequence at end of input"));
1343 /* At this point, everything is valid, and we just need to convert
1345 /********** DIFFERENT for UTF8/UCS4 **********/
1346 result
= g_malloc (n_bytes
+ 4);
1351 while (out
< result
+ n_bytes
)
1356 if (c
>= 0xdc00 && c
< 0xe000) /* low surrogate */
1358 wc
= SURROGATE_VALUE (high_surrogate
, c
);
1361 else if (c
>= 0xd800 && c
< 0xdc00) /* high surrogate */
1369 /********** DIFFERENT for UTF8/UCS4 **********/
1370 *(gunichar
*)out
= wc
;
1371 out
+= sizeof (gunichar
);
1377 /********** DIFFERENT for UTF8/UCS4 **********/
1378 *(gunichar
*)out
= 0;
1381 /********** DIFFERENT for UTF8/UCS4 **********/
1382 *items_written
= (out
- result
) / sizeof (gunichar
);
1386 *items_read
= in
- str
;
1388 return (gunichar
*)result
;
1393 * @str: a UTF-8 encoded string
1394 * @len: the maximum length (number of bytes) of @str to use.
1395 * If @len < 0, then the string is nul-terminated.
1396 * @items_read: location to store number of bytes read, or %NULL.
1397 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1398 * returned in case @str contains a trailing partial
1399 * character. If an error occurs then the index of the
1400 * invalid input is stored here.
1401 * @items_written: location to store number of <type>gunichar2</type> written,
1403 * The value stored here does not include the trailing 0.
1404 * @error: location to store the error occuring, or %NULL to ignore
1405 * errors. Any of the errors in #GConvertError other than
1406 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1408 * Convert a string from UTF-8 to UTF-16. A 0 character will be
1409 * added to the result after the converted text.
1411 * Return value: a pointer to a newly allocated UTF-16 string.
1412 * This value must be freed with g_free(). If an
1413 * error occurs, %NULL will be returned and
1417 g_utf8_to_utf16 (const gchar
*str
,
1420 glong
*items_written
,
1423 gunichar2
*result
= NULL
;
1428 g_return_val_if_fail (str
!= NULL
, NULL
);
1432 while ((len
< 0 || str
+ len
- in
> 0) && *in
)
1434 gunichar wc
= g_utf8_get_char_extended (in
, len
< 0 ? 6 : str
+ len
- in
);
1435 if (wc
& 0x80000000)
1437 if (wc
== (gunichar
)-2)
1442 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_PARTIAL_INPUT
,
1443 _("Partial character sequence at end of input"));
1446 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1447 _("Invalid byte sequence in conversion input"));
1454 else if (wc
< 0xe000)
1456 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1457 _("Invalid sequence in conversion input"));
1461 else if (wc
< 0x10000)
1463 else if (wc
< 0x110000)
1467 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1468 _("Character out of range for UTF-16"));
1473 in
= g_utf8_next_char (in
);
1476 result
= g_new (gunichar2
, n16
+ 1);
1479 for (i
= 0; i
< n16
;)
1481 gunichar wc
= g_utf8_get_char (in
);
1489 result
[i
++] = (wc
- 0x10000) / 0x400 + 0xd800;
1490 result
[i
++] = (wc
- 0x10000) % 0x400 + 0xdc00;
1493 in
= g_utf8_next_char (in
);
1499 *items_written
= n16
;
1503 *items_read
= in
- str
;
1510 * @str: a UCS-4 encoded string
1511 * @len: the maximum length (number of characters) of @str to use.
1512 * If @len < 0, then the string is nul-terminated.
1513 * @items_read: location to store number of bytes read, or %NULL.
1514 * If an error occurs then the index of the invalid input
1516 * @items_written: location to store number of <type>gunichar2</type>
1517 * written, or %NULL. The value stored here does not
1518 * include the trailing 0.
1519 * @error: location to store the error occuring, or %NULL to ignore
1520 * errors. Any of the errors in #GConvertError other than
1521 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1523 * Convert a string from UCS-4 to UTF-16. A 0 character will be
1524 * added to the result after the converted text.
1526 * Return value: a pointer to a newly allocated UTF-16 string.
1527 * This value must be freed with g_free(). If an
1528 * error occurs, %NULL will be returned and
1532 g_ucs4_to_utf16 (const gunichar
*str
,
1535 glong
*items_written
,
1538 gunichar2
*result
= NULL
;
1544 while ((len
< 0 || i
< len
) && str
[i
])
1546 gunichar wc
= str
[i
];
1550 else if (wc
< 0xe000)
1552 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1553 _("Invalid sequence in conversion input"));
1557 else if (wc
< 0x10000)
1559 else if (wc
< 0x110000)
1563 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1564 _("Character out of range for UTF-16"));
1572 result
= g_new (gunichar2
, n16
+ 1);
1574 for (i
= 0, j
= 0; j
< n16
; i
++)
1576 gunichar wc
= str
[i
];
1584 result
[j
++] = (wc
- 0x10000) / 0x400 + 0xd800;
1585 result
[j
++] = (wc
- 0x10000) % 0x400 + 0xdc00;
1591 *items_written
= n16
;
1600 #define CONTINUATION_CHAR \
1602 if ((*(guchar *)p & 0xc0) != 0x80) /* 10xxxxxx */ \
1605 val |= (*(guchar *)p) & 0x3f; \
1608 static const gchar
*
1609 fast_validate (const char *str
)
1616 for (p
= str
; *p
; p
++)
1618 if (*(guchar
*)p
< 128)
1625 if ((*(guchar
*)p
& 0xe0) == 0xc0) /* 110xxxxx */
1627 if (G_UNLIKELY ((*(guchar
*)p
& 0x1e) == 0))
1630 if (G_UNLIKELY ((*(guchar
*)p
& 0xc0) != 0x80)) /* 10xxxxxx */
1635 if ((*(guchar
*)p
& 0xf0) == 0xe0) /* 1110xxxx */
1638 val
= *(guchar
*)p
& 0x0f;
1641 else if ((*(guchar
*)p
& 0xf8) == 0xf0) /* 11110xxx */
1644 val
= *(guchar
*)p
& 0x07;
1657 if (G_UNLIKELY (val
< min
))
1660 if (G_UNLIKELY (!UNICODE_VALID(val
)))
1674 static const gchar
*
1675 fast_validate_len (const char *str
,
1683 g_assert (max_len
>= 0);
1685 for (p
= str
; ((p
- str
) < max_len
) && *p
; p
++)
1687 if (*(guchar
*)p
< 128)
1694 if ((*(guchar
*)p
& 0xe0) == 0xc0) /* 110xxxxx */
1696 if (G_UNLIKELY (max_len
- (p
- str
) < 2))
1699 if (G_UNLIKELY ((*(guchar
*)p
& 0x1e) == 0))
1702 if (G_UNLIKELY ((*(guchar
*)p
& 0xc0) != 0x80)) /* 10xxxxxx */
1707 if ((*(guchar
*)p
& 0xf0) == 0xe0) /* 1110xxxx */
1709 if (G_UNLIKELY (max_len
- (p
- str
) < 3))
1713 val
= *(guchar
*)p
& 0x0f;
1716 else if ((*(guchar
*)p
& 0xf8) == 0xf0) /* 11110xxx */
1718 if (G_UNLIKELY (max_len
- (p
- str
) < 4))
1722 val
= *(guchar
*)p
& 0x07;
1735 if (G_UNLIKELY (val
< min
))
1737 if (G_UNLIKELY (!UNICODE_VALID(val
)))
1753 * @str: a pointer to character data
1754 * @max_len: max bytes to validate, or -1 to go until NUL
1755 * @end: return location for end of valid data
1757 * Validates UTF-8 encoded text. @str is the text to validate;
1758 * if @str is nul-terminated, then @max_len can be -1, otherwise
1759 * @max_len should be the number of bytes to validate.
1760 * If @end is non-%NULL, then the end of the valid range
1761 * will be stored there (i.e. the start of the first invalid
1762 * character if some bytes were invalid, or the end of the text
1763 * being validated otherwise).
1765 * Note that g_utf8_validate() returns %FALSE if @max_len is
1766 * positive and NUL is met before @max_len bytes have been read.
1768 * Returns %TRUE if all of @str was valid. Many GLib and GTK+
1769 * routines <emphasis>require</emphasis> valid UTF-8 as input;
1770 * so data read from a file or the network should be checked
1771 * with g_utf8_validate() before doing anything else with it.
1773 * Return value: %TRUE if the text was valid UTF-8
1776 g_utf8_validate (const char *str
,
1784 p
= fast_validate (str
);
1786 p
= fast_validate_len (str
, max_len
);
1791 if ((max_len
>= 0 && p
!= str
+ max_len
) ||
1792 (max_len
< 0 && *p
!= '\0'))
1799 * g_unichar_validate:
1800 * @ch: a Unicode character
1802 * Checks whether @ch is a valid Unicode character. Some possible
1803 * integer values of @ch will not be valid. 0 is considered a valid
1804 * character, though it's normally a string terminator.
1806 * Return value: %TRUE if @ch is a valid Unicode character
1809 g_unichar_validate (gunichar ch
)
1811 return UNICODE_VALID (ch
);
1815 * g_utf8_strreverse:
1816 * @str: a UTF-8 encoded string
1817 * @len: the maximum length of @str to use, in bytes. If @len < 0,
1818 * then the string is nul-terminated.
1820 * Reverses a UTF-8 string. @str must be valid UTF-8 encoded text.
1821 * (Use g_utf8_validate() on all text before trying to use UTF-8
1822 * utility functions with it.)
1824 * This function is intended for programmatic uses of reversed strings.
1825 * It pays no attention to decomposed characters, combining marks, byte
1826 * order marks, directional indicators (LRM, LRO, etc) and similar
1827 * characters which might need special handling when reversing a string
1828 * for display purposes.
1830 * Note that unlike g_strreverse(), this function returns
1831 * newly-allocated memory, which should be freed with g_free() when
1834 * Returns: a newly-allocated string which is the reverse of @str.
1839 g_utf8_strreverse (const gchar
*str
,
1848 result
= g_new (gchar
, len
+ 1);
1853 gchar
*m
, skip
= g_utf8_skip
[*(guchar
*) p
];
1855 for (m
= r
; skip
; skip
--)
1865 _g_utf8_make_valid (const gchar
*name
)
1868 const gchar
*remainder
, *invalid
;
1869 gint remaining_bytes
, valid_bytes
;
1871 g_return_val_if_fail (name
!= NULL
, NULL
);
1875 remaining_bytes
= strlen (name
);
1877 while (remaining_bytes
!= 0)
1879 if (g_utf8_validate (remainder
, remaining_bytes
, &invalid
))
1881 valid_bytes
= invalid
- remainder
;
1884 string
= g_string_sized_new (remaining_bytes
);
1886 g_string_append_len (string
, remainder
, valid_bytes
);
1887 /* append U+FFFD REPLACEMENT CHARACTER */
1888 g_string_append (string
, "\357\277\275");
1890 remaining_bytes
-= valid_bytes
+ 1;
1891 remainder
= invalid
+ 1;
1895 return g_strdup (name
);
1897 g_string_append (string
, remainder
);
1899 g_assert (g_utf8_validate (string
->str
, -1, NULL
));
1901 return g_string_free (string
, FALSE
);