1 /* gutf8.c - Operations on UTF-8 strings.
3 * Copyright (C) 1999 Tom Tromey
4 * Copyright (C) 2000 Red Hat, Inc.
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 02111-1307, USA.
32 #ifdef G_PLATFORM_WIN32
39 #include "libcharset/libcharset.h"
43 #define UTF8_COMPUTE(Char, Mask, Len) \
49 else if ((Char & 0xe0) == 0xc0) \
54 else if ((Char & 0xf0) == 0xe0) \
59 else if ((Char & 0xf8) == 0xf0) \
64 else if ((Char & 0xfc) == 0xf8) \
69 else if ((Char & 0xfe) == 0xfc) \
77 #define UTF8_LENGTH(Char) \
78 ((Char) < 0x80 ? 1 : \
79 ((Char) < 0x800 ? 2 : \
80 ((Char) < 0x10000 ? 3 : \
81 ((Char) < 0x200000 ? 4 : \
82 ((Char) < 0x4000000 ? 5 : 6)))))
85 #define UTF8_GET(Result, Chars, Count, Mask, Len) \
86 (Result) = (Chars)[0] & (Mask); \
87 for ((Count) = 1; (Count) < (Len); ++(Count)) \
89 if (((Chars)[(Count)] & 0xc0) != 0x80) \
95 (Result) |= ((Chars)[(Count)] & 0x3f); \
98 #define UNICODE_VALID(Char) \
99 ((Char) < 0x110000 && \
100 ((Char) < 0xD800 || (Char) >= 0xE000) && \
101 (Char) != 0xFFFE && (Char) != 0xFFFF)
104 static const gchar utf8_skip_data
[256] = {
105 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
106 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
107 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
108 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
109 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
110 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
111 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
112 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,0,0
115 const gchar
* const g_utf8_skip
= utf8_skip_data
;
118 * g_utf8_find_prev_char:
119 * @str: pointer to the beginning of a UTF-8 encoded string
120 * @p: pointer to some position within @str
122 * Given a position @p with a UTF-8 encoded string @str, find the start
123 * of the previous UTF-8 character starting before @p. Returns %NULL if no
124 * UTF-8 characters are present in @p before @str.
126 * @p does not have to be at the beginning of a UTF-8 character. No check
127 * is made to see if the character found is actually valid other than
128 * it starts with an appropriate byte.
130 * Return value: a pointer to the found character or %NULL.
133 g_utf8_find_prev_char (const char *str
,
136 for (--p
; p
>= str
; --p
)
138 if ((*p
& 0xc0) != 0x80)
145 * g_utf8_find_next_char:
146 * @p: a pointer to a position within a UTF-8 encoded string
147 * @end: a pointer to the end of the string, or %NULL to indicate
148 * that the string is nul-terminated, in which case
149 * the returned value will be
151 * Finds the start of the next UTF-8 character in the string after @p.
153 * @p does not have to be at the beginning of a UTF-8 character. No check
154 * is made to see if the character found is actually valid other than
155 * it starts with an appropriate byte.
157 * Return value: a pointer to the found character or %NULL
160 g_utf8_find_next_char (const gchar
*p
,
166 for (++p
; p
< end
&& (*p
& 0xc0) == 0x80; ++p
)
169 for (++p
; (*p
& 0xc0) == 0x80; ++p
)
172 return (p
== end
) ? NULL
: (gchar
*)p
;
177 * @p: a pointer to a position within a UTF-8 encoded string
179 * Finds the previous UTF-8 character in the string before @p.
181 * @p does not have to be at the beginning of a UTF-8 character. No check
182 * is made to see if the character found is actually valid other than
183 * it starts with an appropriate byte. If @p might be the first
184 * character of the string, you must use g_utf8_find_prev_char() instead.
186 * Return value: a pointer to the found character.
189 g_utf8_prev_char (const gchar
*p
)
194 if ((*p
& 0xc0) != 0x80)
201 * @p: pointer to the start of a UTF-8 encoded string.
202 * @max: the maximum number of bytes to examine. If @max
203 * is less than 0, then the string is assumed to be
206 * Returns the length of the string in characters.
208 * Return value: the length of the string in characters
211 g_utf8_strlen (const gchar
*p
,
215 const gchar
*start
= p
;
221 p
= g_utf8_next_char (p
);
230 p
= g_utf8_next_char (p
);
232 while (p
- start
< max
&& *p
)
235 p
= g_utf8_next_char (p
);
238 /* only do the last len increment if we got a complete
239 * char (don't count partial chars)
241 if (p
- start
== max
)
250 * @p: a pointer to Unicode character encoded as UTF-8
252 * Converts a sequence of bytes encoded as UTF-8 to a Unicode character.
253 * If @p does not point to a valid UTF-8 encoded character, results are
254 * undefined. If you are not sure that the bytes are complete
255 * valid Unicode characters, you should use g_utf8_get_char_validated()
258 * Return value: the resulting character
261 g_utf8_get_char (const gchar
*p
)
263 int i
, mask
= 0, len
;
265 unsigned char c
= (unsigned char) *p
;
267 UTF8_COMPUTE (c
, mask
, len
);
270 UTF8_GET (result
, p
, i
, mask
, len
);
276 * g_utf8_offset_to_pointer:
277 * @str: a UTF-8 encoded string
278 * @offset: a character offset within @str
280 * Converts from an integer character offset to a pointer to a position
283 * Return value: the resulting pointer
286 g_utf8_offset_to_pointer (const gchar
*str
,
289 const gchar
*s
= str
;
291 s
= g_utf8_next_char (s
);
297 * g_utf8_pointer_to_offset:
298 * @str: a UTF-8 encoded string
299 * @pos: a pointer to a position within @str
301 * Converts from a pointer to position within a string to a integer
304 * Return value: the resulting character offset
307 g_utf8_pointer_to_offset (const gchar
*str
,
310 const gchar
*s
= str
;
315 s
= g_utf8_next_char (s
);
325 * @dest: buffer to fill with characters from @src
326 * @src: UTF-8 encoded string
327 * @n: character count
329 * Like the standard C <function>strncpy()</function> function, but
330 * copies a given number of characters instead of a given number of
331 * bytes. The @src string must be valid UTF-8 encoded text.
332 * (Use g_utf8_validate() on all text before trying to use UTF-8
333 * utility functions with it.)
335 * Return value: @dest
338 g_utf8_strncpy (gchar
*dest
,
342 const gchar
*s
= src
;
345 s
= g_utf8_next_char(s
);
348 strncpy(dest
, src
, s
- src
);
353 G_LOCK_DEFINE_STATIC (aliases
);
356 get_alias_hash (void)
358 static GHashTable
*alias_hash
= NULL
;
365 alias_hash
= g_hash_table_new (g_str_hash
, g_str_equal
);
367 aliases
= _g_locale_get_charset_aliases ();
368 while (*aliases
!= '\0')
370 const char *canonical
;
372 const char **alias_array
;
376 aliases
+= strlen (aliases
) + 1;
378 aliases
+= strlen (aliases
) + 1;
380 alias_array
= g_hash_table_lookup (alias_hash
, canonical
);
383 while (alias_array
[count
])
387 alias_array
= g_renew (const char *, alias_array
, count
+ 2);
388 alias_array
[count
] = alias
;
389 alias_array
[count
+ 1] = NULL
;
391 g_hash_table_insert (alias_hash
, (char *)canonical
, alias_array
);
400 /* As an abuse of the alias table, the following routines gets
401 * the charsets that are aliases for the canonical name.
404 _g_charset_get_aliases (const char *canonical_name
)
406 GHashTable
*alias_hash
= get_alias_hash ();
408 return g_hash_table_lookup (alias_hash
, canonical_name
);
412 g_utf8_get_charset_internal (const char **a
)
414 const char *charset
= getenv("CHARSET");
416 if (charset
&& *charset
)
420 if (charset
&& strstr (charset
, "UTF-8"))
426 /* The libcharset code tries to be thread-safe without
427 * a lock, but has a memory leak and a missing memory
428 * barrier, so we lock for it
431 charset
= _g_locale_charset ();
434 if (charset
&& *charset
)
438 if (charset
&& strstr (charset
, "UTF-8"))
444 /* Assume this for compatibility at present. */
450 static int utf8_locale_cache
= -1;
451 static const char *utf8_charset_cache
= NULL
;
455 * @charset: return location for character set name
457 * Obtains the character set for the current locale; you might use
458 * this character set as an argument to g_convert(), to convert from
459 * the current locale's encoding to some other encoding. (Frequently
460 * g_locale_to_utf8() and g_locale_from_utf8() are nice shortcuts,
463 * The return value is %TRUE if the locale's encoding is UTF-8, in that
464 * case you can perhaps avoid calling g_convert().
466 * The string returned in @charset is not allocated, and should not be
469 * Return value: %TRUE if the returned charset is UTF-8
472 g_get_charset (G_CONST_RETURN
char **charset
)
474 if (utf8_locale_cache
!= -1)
477 *charset
= utf8_charset_cache
;
478 return utf8_locale_cache
;
480 utf8_locale_cache
= g_utf8_get_charset_internal (&utf8_charset_cache
);
482 *charset
= utf8_charset_cache
;
483 return utf8_locale_cache
;
490 * @c: a ISO10646 character code
491 * @outbuf: output buffer, must have at least 6 bytes of space.
492 * If %NULL, the length will be computed and returned
493 * and nothing will be written to @outbuf.
495 * Converts a single character to UTF-8.
497 * Return value: number of bytes written
500 g_unichar_to_utf8 (gunichar c
,
517 else if (c
< 0x10000)
522 else if (c
< 0x200000)
527 else if (c
< 0x4000000)
540 for (i
= len
- 1; i
> 0; --i
)
542 outbuf
[i
] = (c
& 0x3f) | 0x80;
545 outbuf
[0] = c
| first
;
553 * @p: a nul-terminated UTF-8 encoded string
554 * @len: the maximum length of @p
555 * @c: a ISO10646 character
557 * Finds the leftmost occurrence of the given ISO10646 character
558 * in a UTF-8 encoded string, while limiting the search to @len bytes.
559 * If @len is -1, allow unbounded search.
561 * Return value: %NULL if the string does not contain the character,
562 * otherwise, a pointer to the start of the leftmost occurrence of
563 * the character in the string.
566 g_utf8_strchr (const char *p
,
572 gint charlen
= g_unichar_to_utf8 (c
, ch
);
575 return g_strstr_len (p
, len
, ch
);
581 * @p: a nul-terminated UTF-8 encoded string
582 * @len: the maximum length of @p
583 * @c: a ISO10646 character
585 * Find the rightmost occurrence of the given ISO10646 character
586 * in a UTF-8 encoded string, while limiting the search to @len bytes.
587 * If @len is -1, allow unbounded search.
589 * Return value: %NULL if the string does not contain the character,
590 * otherwise, a pointer to the start of the rightmost occurrence of the
591 * character in the string.
594 g_utf8_strrchr (const char *p
,
600 gint charlen
= g_unichar_to_utf8 (c
, ch
);
603 return g_strrstr_len (p
, len
, ch
);
607 /* Like g_utf8_get_char, but take a maximum length
608 * and return (gunichar)-2 on incomplete trailing character
610 static inline gunichar
611 g_utf8_get_char_extended (const gchar
*p
,
615 gunichar wc
= (guchar
) *p
;
655 if (max_len
>= 0 && len
> max_len
)
657 for (i
= 1; i
< max_len
; i
++)
659 if ((((guchar
*)p
)[i
] & 0xc0) != 0x80)
665 for (i
= 1; i
< len
; ++i
)
667 gunichar ch
= ((guchar
*)p
)[i
];
669 if ((ch
& 0xc0) != 0x80)
681 if (UTF8_LENGTH(wc
) != len
)
688 * g_utf8_get_char_validated:
689 * @p: a pointer to Unicode character encoded as UTF-8
690 * @max_len: the maximum number of bytes to read, or -1, for no maximum.
692 * Convert a sequence of bytes encoded as UTF-8 to a Unicode character.
693 * This function checks for incomplete characters, for invalid characters
694 * such as characters that are out of the range of Unicode, and for
695 * overlong encodings of valid characters.
697 * Return value: the resulting character. If @p points to a partial
698 * sequence at the end of a string that could begin a valid character,
699 * returns (gunichar)-2; otherwise, if @p does not point to a valid
700 * UTF-8 encoded Unicode character, returns (gunichar)-1.
703 g_utf8_get_char_validated (const gchar
*p
,
706 gunichar result
= g_utf8_get_char_extended (p
, max_len
);
708 if (result
& 0x80000000)
710 else if (!UNICODE_VALID (result
))
717 * g_utf8_to_ucs4_fast:
718 * @str: a UTF-8 encoded string
719 * @len: the maximum length of @str to use. If @len < 0, then
720 * the string is nul-terminated.
721 * @items_written: location to store the number of characters in the
724 * Convert a string from UTF-8 to a 32-bit fixed width
725 * representation as UCS-4, assuming valid UTF-8 input.
726 * This function is roughly twice as fast as g_utf8_to_ucs4()
727 * but does no error checking on the input.
729 * Return value: a pointer to a newly allocated UCS-4 string.
730 * This value must be freed with g_free().
733 g_utf8_to_ucs4_fast (const gchar
*str
,
735 glong
*items_written
)
742 g_return_val_if_fail (str
!= NULL
, NULL
);
750 p
= g_utf8_next_char (p
);
756 while (p
< str
+ len
&& *p
)
758 p
= g_utf8_next_char (p
);
763 result
= g_new (gunichar
, n_chars
+ 1);
766 for (i
=0; i
< n_chars
; i
++)
768 gunichar wc
= ((unsigned char *)p
)[0];
803 for (j
= 1; j
< charlen
; j
++)
806 wc
|= ((unsigned char *)p
)[j
] & 0x3f;
823 * @str: a UTF-8 encoded string
824 * @len: the maximum length of @str to use. If @len < 0, then
825 * the string is nul-terminated.
826 * @items_read: location to store number of bytes read, or %NULL.
827 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
828 * returned in case @str contains a trailing partial
829 * character. If an error occurs then the index of the
830 * invalid input is stored here.
831 * @items_written: location to store number of characters written or %NULL.
832 * The value here stored does not include the trailing 0
834 * @error: location to store the error occuring, or %NULL to ignore
835 * errors. Any of the errors in #GConvertError other than
836 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
838 * Convert a string from UTF-8 to a 32-bit fixed width
839 * representation as UCS-4. A trailing 0 will be added to the
840 * string after the converted text.
842 * Return value: a pointer to a newly allocated UCS-4 string.
843 * This value must be freed with g_free(). If an
844 * error occurs, %NULL will be returned and
848 g_utf8_to_ucs4 (const gchar
*str
,
851 glong
*items_written
,
854 gunichar
*result
= NULL
;
860 while ((len
< 0 || str
+ len
- in
> 0) && *in
)
862 gunichar wc
= g_utf8_get_char_extended (in
, str
+ len
- in
);
865 if (wc
== (gunichar
)-2)
870 g_set_error (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_PARTIAL_INPUT
,
871 _("Partial character sequence at end of input"));
874 g_set_error (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
875 _("Invalid byte sequence in conversion input"));
882 in
= g_utf8_next_char (in
);
885 result
= g_new (gunichar
, n_chars
+ 1);
888 for (i
=0; i
< n_chars
; i
++)
890 result
[i
] = g_utf8_get_char (in
);
891 in
= g_utf8_next_char (in
);
896 *items_written
= n_chars
;
900 *items_read
= in
- str
;
907 * @str: a UCS-4 encoded string
908 * @len: the maximum length of @str to use. If @len < 0, then
909 * the string is terminated with a 0 character.
910 * @items_read: location to store number of characters read read, or %NULL.
911 * @items_written: location to store number of bytes written or %NULL.
912 * The value here stored does not include the trailing 0
914 * @error: location to store the error occuring, or %NULL to ignore
915 * errors. Any of the errors in #GConvertError other than
916 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
918 * Convert a string from a 32-bit fixed width representation as UCS-4.
919 * to UTF-8. The result will be terminated with a 0 byte.
921 * Return value: a pointer to a newly allocated UTF-8 string.
922 * This value must be freed with g_free(). If an
923 * error occurs, %NULL will be returned and
927 g_ucs4_to_utf8 (const gunichar
*str
,
930 glong
*items_written
,
934 gchar
*result
= NULL
;
939 for (i
= 0; len
< 0 || i
< len
; i
++)
944 if (str
[i
] >= 0x80000000)
949 g_set_error (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
950 _("Character out of range for UTF-8"));
954 result_length
+= UTF8_LENGTH (str
[i
]);
957 result
= g_malloc (result_length
+ 1);
961 while (p
< result
+ result_length
)
962 p
+= g_unichar_to_utf8 (str
[i
++], p
);
967 *items_written
= p
- result
;
976 #define SURROGATE_VALUE(h,l) (((h) - 0xd800) * 0x400 + (l) - 0xdc00 + 0x10000)
980 * @str: a UTF-16 encoded string
981 * @len: the maximum length of @str to use. If @len < 0, then
982 * the string is terminated with a 0 character.
983 * @items_read: location to store number of words read, or %NULL.
984 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
985 * returned in case @str contains a trailing partial
986 * character. If an error occurs then the index of the
987 * invalid input is stored here.
988 * @items_written: location to store number of bytes written, or %NULL.
989 * The value stored here does not include the trailing
991 * @error: location to store the error occuring, or %NULL to ignore
992 * errors. Any of the errors in #GConvertError other than
993 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
995 * Convert a string from UTF-16 to UTF-8. The result will be
996 * terminated with a 0 byte.
998 * Return value: a pointer to a newly allocated UTF-8 string.
999 * This value must be freed with g_free(). If an
1000 * error occurs, %NULL will be returned and
1004 g_utf16_to_utf8 (const gunichar2
*str
,
1007 glong
*items_written
,
1010 /* This function and g_utf16_to_ucs4 are almost exactly identical - The lines that differ
1013 const gunichar2
*in
;
1015 gchar
*result
= NULL
;
1017 gunichar high_surrogate
;
1019 g_return_val_if_fail (str
!= 0, NULL
);
1024 while ((len
< 0 || in
- str
< len
) && *in
)
1029 if (c
>= 0xdc00 && c
< 0xe000) /* low surrogate */
1033 wc
= SURROGATE_VALUE (high_surrogate
, c
);
1038 g_set_error (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1039 _("Invalid sequence in conversion input"));
1047 g_set_error (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1048 _("Invalid sequence in conversion input"));
1052 if (c
>= 0xd800 && c
< 0xdc00) /* high surrogate */
1061 /********** DIFFERENT for UTF8/UCS4 **********/
1062 n_bytes
+= UTF8_LENGTH (wc
);
1068 if (high_surrogate
&& !items_read
)
1070 g_set_error (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_PARTIAL_INPUT
,
1071 _("Partial character sequence at end of input"));
1075 /* At this point, everything is valid, and we just need to convert
1077 /********** DIFFERENT for UTF8/UCS4 **********/
1078 result
= g_malloc (n_bytes
+ 1);
1083 while (out
< result
+ n_bytes
)
1088 if (c
>= 0xdc00 && c
< 0xe000) /* low surrogate */
1090 wc
= SURROGATE_VALUE (high_surrogate
, c
);
1093 else if (c
>= 0xd800 && c
< 0xdc00) /* high surrogate */
1101 /********** DIFFERENT for UTF8/UCS4 **********/
1102 out
+= g_unichar_to_utf8 (wc
, out
);
1108 /********** DIFFERENT for UTF8/UCS4 **********/
1112 /********** DIFFERENT for UTF8/UCS4 **********/
1113 *items_written
= out
- result
;
1117 *items_read
= in
- str
;
1124 * @str: a UTF-16 encoded string
1125 * @len: the maximum length of @str to use. If @len < 0, then
1126 * the string is terminated with a 0 character.
1127 * @items_read: location to store number of words read, or %NULL.
1128 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1129 * returned in case @str contains a trailing partial
1130 * character. If an error occurs then the index of the
1131 * invalid input is stored here.
1132 * @items_written: location to store number of characters written, or %NULL.
1133 * The value stored here does not include the trailing
1135 * @error: location to store the error occuring, or %NULL to ignore
1136 * errors. Any of the errors in #GConvertError other than
1137 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1139 * Convert a string from UTF-16 to UCS-4. The result will be
1140 * terminated with a 0 character.
1142 * Return value: a pointer to a newly allocated UCS-4 string.
1143 * This value must be freed with g_free(). If an
1144 * error occurs, %NULL will be returned and
1148 g_utf16_to_ucs4 (const gunichar2
*str
,
1151 glong
*items_written
,
1154 const gunichar2
*in
;
1156 gchar
*result
= NULL
;
1158 gunichar high_surrogate
;
1160 g_return_val_if_fail (str
!= 0, NULL
);
1165 while ((len
< 0 || in
- str
< len
) && *in
)
1170 if (c
>= 0xdc00 && c
< 0xe000) /* low surrogate */
1174 wc
= SURROGATE_VALUE (high_surrogate
, c
);
1179 g_set_error (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1180 _("Invalid sequence in conversion input"));
1188 g_set_error (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1189 _("Invalid sequence in conversion input"));
1193 if (c
>= 0xd800 && c
< 0xdc00) /* high surrogate */
1202 /********** DIFFERENT for UTF8/UCS4 **********/
1203 n_bytes
+= sizeof (gunichar
);
1209 if (high_surrogate
&& !items_read
)
1211 g_set_error (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_PARTIAL_INPUT
,
1212 _("Partial character sequence at end of input"));
1216 /* At this point, everything is valid, and we just need to convert
1218 /********** DIFFERENT for UTF8/UCS4 **********/
1219 result
= g_malloc (n_bytes
+ 4);
1224 while (out
< result
+ n_bytes
)
1229 if (c
>= 0xdc00 && c
< 0xe000) /* low surrogate */
1231 wc
= SURROGATE_VALUE (high_surrogate
, c
);
1234 else if (c
>= 0xd800 && c
< 0xdc00) /* high surrogate */
1242 /********** DIFFERENT for UTF8/UCS4 **********/
1243 *(gunichar
*)out
= wc
;
1244 out
+= sizeof (gunichar
);
1250 /********** DIFFERENT for UTF8/UCS4 **********/
1251 *(gunichar
*)out
= 0;
1254 /********** DIFFERENT for UTF8/UCS4 **********/
1255 *items_written
= (out
- result
) / sizeof (gunichar
);
1259 *items_read
= in
- str
;
1261 return (gunichar
*)result
;
1266 * @str: a UTF-8 encoded string
1267 * @len: the maximum length of @str to use. If @len < 0, then
1268 * the string is nul-terminated.
1270 * @items_read: location to store number of bytes read, or %NULL.
1271 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1272 * returned in case @str contains a trailing partial
1273 * character. If an error occurs then the index of the
1274 * invalid input is stored here.
1275 * @items_written: location to store number of words written, or %NULL.
1276 * The value stored here does not include the trailing
1278 * @error: location to store the error occuring, or %NULL to ignore
1279 * errors. Any of the errors in #GConvertError other than
1280 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1282 * Convert a string from UTF-8 to UTF-16. A 0 word will be
1283 * added to the result after the converted text.
1285 * Return value: a pointer to a newly allocated UTF-16 string.
1286 * This value must be freed with g_free(). If an
1287 * error occurs, %NULL will be returned and
1291 g_utf8_to_utf16 (const gchar
*str
,
1294 glong
*items_written
,
1297 gunichar2
*result
= NULL
;
1302 g_return_val_if_fail (str
!= NULL
, NULL
);
1306 while ((len
< 0 || str
+ len
- in
> 0) && *in
)
1308 gunichar wc
= g_utf8_get_char_extended (in
, str
+ len
- in
);
1309 if (wc
& 0x80000000)
1311 if (wc
== (gunichar
)-2)
1316 g_set_error (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_PARTIAL_INPUT
,
1317 _("Partial character sequence at end of input"));
1320 g_set_error (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1321 _("Invalid byte sequence in conversion input"));
1328 else if (wc
< 0xe000)
1330 g_set_error (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1331 _("Invalid sequence in conversion input"));
1335 else if (wc
< 0x10000)
1337 else if (wc
< 0x110000)
1341 g_set_error (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1342 _("Character out of range for UTF-16"));
1347 in
= g_utf8_next_char (in
);
1350 result
= g_new (gunichar2
, n16
+ 1);
1353 for (i
= 0; i
< n16
;)
1355 gunichar wc
= g_utf8_get_char (in
);
1363 result
[i
++] = (wc
- 0x10000) / 0x400 + 0xd800;
1364 result
[i
++] = (wc
- 0x10000) % 0x400 + 0xdc00;
1367 in
= g_utf8_next_char (in
);
1373 *items_written
= n16
;
1377 *items_read
= in
- str
;
1384 * @str: a UCS-4 encoded string
1385 * @len: the maximum length of @str to use. If @len < 0, then
1386 * the string is terminated with a 0 character.
1387 * @items_read: location to store number of bytes read, or %NULL.
1388 * If an error occurs then the index of the invalid input
1390 * @items_written: location to store number of words written, or %NULL.
1391 * The value stored here does not include the trailing
1393 * @error: location to store the error occuring, or %NULL to ignore
1394 * errors. Any of the errors in #GConvertError other than
1395 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1397 * Convert a string from UCS-4 to UTF-16. A 0 word will be
1398 * added to the result after the converted text.
1400 * Return value: a pointer to a newly allocated UTF-16 string.
1401 * This value must be freed with g_free(). If an
1402 * error occurs, %NULL will be returned and
1406 g_ucs4_to_utf16 (const gunichar
*str
,
1409 glong
*items_written
,
1412 gunichar2
*result
= NULL
;
1418 while ((len
< 0 || i
< len
) && str
[i
])
1420 gunichar wc
= str
[i
];
1424 else if (wc
< 0xe000)
1426 g_set_error (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1427 _("Invalid sequence in conversion input"));
1431 else if (wc
< 0x10000)
1433 else if (wc
< 0x110000)
1437 g_set_error (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1438 _("Character out of range for UTF-16"));
1446 result
= g_new (gunichar2
, n16
+ 1);
1448 for (i
= 0, j
= 0; j
< n16
; i
++)
1450 gunichar wc
= str
[i
];
1458 result
[j
++] = (wc
- 0x10000) / 0x400 + 0xd800;
1459 result
[j
++] = (wc
- 0x10000) % 0x400 + 0xdc00;
1465 *items_written
= n16
;
1476 * @str: a pointer to character data
1477 * @max_len: max bytes to validate, or -1 to go until nul
1478 * @end: return location for end of valid data
1480 * Validates UTF-8 encoded text. @str is the text to validate;
1481 * if @str is nul-terminated, then @max_len can be -1, otherwise
1482 * @max_len should be the number of bytes to validate.
1483 * If @end is non-%NULL, then the end of the valid range
1484 * will be stored there (i.e. the address of the first invalid byte
1485 * if some bytes were invalid, or the end of the text being validated
1488 * Returns %TRUE if all of @str was valid. Many GLib and GTK+
1489 * routines <emphasis>require</emphasis> valid UTF-8 as input;
1490 * so data read from a file or the network should be checked
1491 * with g_utf8_validate() before doing anything else with it.
1493 * Return value: %TRUE if the text was valid UTF-8
1496 g_utf8_validate (const gchar
*str
,
1503 g_return_val_if_fail (str
!= NULL
, FALSE
);
1510 while ((max_len
< 0 || (p
- str
) < max_len
) && *p
)
1512 int i
, mask
= 0, len
;
1514 unsigned char c
= (unsigned char) *p
;
1516 UTF8_COMPUTE (c
, mask
, len
);
1521 /* check that the expected number of bytes exists in str */
1523 ((max_len
- (p
- str
)) < len
))
1526 UTF8_GET (result
, p
, i
, mask
, len
);
1528 if (UTF8_LENGTH (result
) != len
) /* Check for overlong UTF-8 */
1531 if (result
== (gunichar
)-1)
1534 if (!UNICODE_VALID (result
))
1543 /* See that we covered the entire length if a length was
1544 * passed in, or that we ended on a nul if not
1547 p
!= (str
+ max_len
))
1549 else if (max_len
< 0 &&
1557 * g_unichar_validate:
1558 * @ch: a Unicode character
1560 * Checks whether @ch is a valid Unicode character. Some possible
1561 * integer values of @ch will not be valid. 0 is considered a valid
1562 * character, though it's normally a string terminator.
1564 * Return value: %TRUE if @ch is a valid Unicode character
1567 g_unichar_validate (gunichar ch
)
1569 return UNICODE_VALID (ch
);