1 /* gutf8.c - Operations on UTF-8 strings.
3 * Copyright (C) 1999 Tom Tromey
4 * Copyright (C) 2000 Red Hat, Inc.
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 02111-1307, USA.
32 #ifdef G_PLATFORM_WIN32
39 #include "libcharset/libcharset.h"
44 #define UTF8_COMPUTE(Char, Mask, Len) \
50 else if ((Char & 0xe0) == 0xc0) \
55 else if ((Char & 0xf0) == 0xe0) \
60 else if ((Char & 0xf8) == 0xf0) \
65 else if ((Char & 0xfc) == 0xf8) \
70 else if ((Char & 0xfe) == 0xfc) \
78 #define UTF8_LENGTH(Char) \
79 ((Char) < 0x80 ? 1 : \
80 ((Char) < 0x800 ? 2 : \
81 ((Char) < 0x10000 ? 3 : \
82 ((Char) < 0x200000 ? 4 : \
83 ((Char) < 0x4000000 ? 5 : 6)))))
86 #define UTF8_GET(Result, Chars, Count, Mask, Len) \
87 (Result) = (Chars)[0] & (Mask); \
88 for ((Count) = 1; (Count) < (Len); ++(Count)) \
90 if (((Chars)[(Count)] & 0xc0) != 0x80) \
96 (Result) |= ((Chars)[(Count)] & 0x3f); \
100 * Check whether a Unicode (5.2) char is in a valid range.
102 * The first check comes from the Unicode guarantee to never encode
103 * a point above 0x0010ffff, since UTF-16 couldn't represent it.
105 * The second check covers surrogate pairs (category Cs).
107 * The last two checks cover "Noncharacter": defined as:
108 * "A code point that is permanently reserved for
109 * internal use, and that should never be interchanged. In
110 * Unicode 3.1, these consist of the values U+nFFFE and U+nFFFF
111 * (where n is from 0 to 10_16) and the values U+FDD0..U+FDEF."
113 * @param Char the character
115 #define UNICODE_VALID(Char) \
116 ((Char) < 0x110000 && \
117 (((Char) & 0xFFFFF800) != 0xD800) && \
118 ((Char) < 0xFDD0 || (Char) > 0xFDEF) && \
119 ((Char) & 0xFFFE) != 0xFFFE)
122 static const gchar utf8_skip_data
[256] = {
123 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
124 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
125 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
126 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
127 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
128 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
129 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
130 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,1,1
133 const gchar
* const g_utf8_skip
= utf8_skip_data
;
136 * g_utf8_find_prev_char:
137 * @str: pointer to the beginning of a UTF-8 encoded string
138 * @p: pointer to some position within @str
140 * Given a position @p with a UTF-8 encoded string @str, find the start
141 * of the previous UTF-8 character starting before @p. Returns %NULL if no
142 * UTF-8 characters are present in @str before @p.
144 * @p does not have to be at the beginning of a UTF-8 character. No check
145 * is made to see if the character found is actually valid other than
146 * it starts with an appropriate byte.
148 * Return value: a pointer to the found character or %NULL.
151 g_utf8_find_prev_char (const char *str
,
154 for (--p
; p
>= str
; --p
)
156 if ((*p
& 0xc0) != 0x80)
163 * g_utf8_find_next_char:
164 * @p: a pointer to a position within a UTF-8 encoded string
165 * @end: a pointer to the byte following the end of the string,
166 * or %NULL to indicate that the string is nul-terminated.
168 * Finds the start of the next UTF-8 character in the string after @p.
170 * @p does not have to be at the beginning of a UTF-8 character. No check
171 * is made to see if the character found is actually valid other than
172 * it starts with an appropriate byte.
174 * Return value: a pointer to the found character or %NULL
177 g_utf8_find_next_char (const gchar
*p
,
183 for (++p
; p
< end
&& (*p
& 0xc0) == 0x80; ++p
)
186 for (++p
; (*p
& 0xc0) == 0x80; ++p
)
189 return (p
== end
) ? NULL
: (gchar
*)p
;
194 * @p: a pointer to a position within a UTF-8 encoded string
196 * Finds the previous UTF-8 character in the string before @p.
198 * @p does not have to be at the beginning of a UTF-8 character. No check
199 * is made to see if the character found is actually valid other than
200 * it starts with an appropriate byte. If @p might be the first
201 * character of the string, you must use g_utf8_find_prev_char() instead.
203 * Return value: a pointer to the found character.
206 g_utf8_prev_char (const gchar
*p
)
211 if ((*p
& 0xc0) != 0x80)
218 * @p: pointer to the start of a UTF-8 encoded string
219 * @max: the maximum number of bytes to examine. If @max
220 * is less than 0, then the string is assumed to be
221 * nul-terminated. If @max is 0, @p will not be examined and
224 * Computes the length of the string in characters, not including
225 * the terminating nul character.
227 * Return value: the length of the string in characters
230 g_utf8_strlen (const gchar
*p
,
234 const gchar
*start
= p
;
235 g_return_val_if_fail (p
!= NULL
|| max
== 0, 0);
241 p
= g_utf8_next_char (p
);
250 p
= g_utf8_next_char (p
);
252 while (p
- start
< max
&& *p
)
255 p
= g_utf8_next_char (p
);
258 /* only do the last len increment if we got a complete
259 * char (don't count partial chars)
261 if (p
- start
<= max
)
270 * @p: a pointer to Unicode character encoded as UTF-8
272 * Converts a sequence of bytes encoded as UTF-8 to a Unicode character.
273 * If @p does not point to a valid UTF-8 encoded character, results are
274 * undefined. If you are not sure that the bytes are complete
275 * valid Unicode characters, you should use g_utf8_get_char_validated()
278 * Return value: the resulting character
281 g_utf8_get_char (const gchar
*p
)
283 int i
, mask
= 0, len
;
285 unsigned char c
= (unsigned char) *p
;
287 UTF8_COMPUTE (c
, mask
, len
);
290 UTF8_GET (result
, p
, i
, mask
, len
);
296 * g_utf8_offset_to_pointer:
297 * @str: a UTF-8 encoded string
298 * @offset: a character offset within @str
300 * Converts from an integer character offset to a pointer to a position
303 * Since 2.10, this function allows to pass a negative @offset to
304 * step backwards. It is usually worth stepping backwards from the end
305 * instead of forwards if @offset is in the last fourth of the string,
306 * since moving forward is about 3 times faster than moving backward.
309 * This function doesn't abort when reaching the end of @str. Therefore
310 * you should be sure that @offset is within string boundaries before
311 * calling that function. Call g_utf8_strlen() when unsure.
313 * This limitation exists as this function is called frequently during
314 * text rendering and therefore has to be as fast as possible.
317 * Return value: the resulting pointer
320 g_utf8_offset_to_pointer (const gchar
*str
,
323 const gchar
*s
= str
;
327 s
= g_utf8_next_char (s
);
332 /* This nice technique for fast backwards stepping
333 * through a UTF-8 string was dubbed "stutter stepping"
334 * by its inventor, Larry Ewing.
340 while ((*s
& 0xc0) == 0x80)
343 offset
+= g_utf8_pointer_to_offset (s
, s1
);
351 * g_utf8_pointer_to_offset:
352 * @str: a UTF-8 encoded string
353 * @pos: a pointer to a position within @str
355 * Converts from a pointer to position within a string to a integer
358 * Since 2.10, this function allows @pos to be before @str, and returns
359 * a negative offset in this case.
361 * Return value: the resulting character offset
364 g_utf8_pointer_to_offset (const gchar
*str
,
367 const gchar
*s
= str
;
371 offset
= - g_utf8_pointer_to_offset (pos
, str
);
375 s
= g_utf8_next_char (s
);
385 * @dest: buffer to fill with characters from @src
386 * @src: UTF-8 encoded string
387 * @n: character count
389 * Like the standard C strncpy() function, but
390 * copies a given number of characters instead of a given number of
391 * bytes. The @src string must be valid UTF-8 encoded text.
392 * (Use g_utf8_validate() on all text before trying to use UTF-8
393 * utility functions with it.)
395 * Return value: @dest
398 g_utf8_strncpy (gchar
*dest
,
402 const gchar
*s
= src
;
405 s
= g_utf8_next_char(s
);
408 strncpy(dest
, src
, s
- src
);
413 G_LOCK_DEFINE_STATIC (aliases
);
416 get_alias_hash (void)
418 static GHashTable
*alias_hash
= NULL
;
425 alias_hash
= g_hash_table_new (g_str_hash
, g_str_equal
);
427 aliases
= _g_locale_get_charset_aliases ();
428 while (*aliases
!= '\0')
430 const char *canonical
;
432 const char **alias_array
;
436 aliases
+= strlen (aliases
) + 1;
438 aliases
+= strlen (aliases
) + 1;
440 alias_array
= g_hash_table_lookup (alias_hash
, canonical
);
443 while (alias_array
[count
])
447 alias_array
= g_renew (const char *, alias_array
, count
+ 2);
448 alias_array
[count
] = alias
;
449 alias_array
[count
+ 1] = NULL
;
451 g_hash_table_insert (alias_hash
, (char *)canonical
, alias_array
);
460 /* As an abuse of the alias table, the following routines gets
461 * the charsets that are aliases for the canonical name.
463 G_GNUC_INTERNAL
const char **
464 _g_charset_get_aliases (const char *canonical_name
)
466 GHashTable
*alias_hash
= get_alias_hash ();
468 return g_hash_table_lookup (alias_hash
, canonical_name
);
472 g_utf8_get_charset_internal (const char *raw_data
,
475 const char *charset
= getenv("CHARSET");
477 if (charset
&& *charset
)
481 if (charset
&& strstr (charset
, "UTF-8"))
487 /* The libcharset code tries to be thread-safe without
488 * a lock, but has a memory leak and a missing memory
489 * barrier, so we lock for it
492 charset
= _g_locale_charset_unalias (raw_data
);
495 if (charset
&& *charset
)
499 if (charset
&& strstr (charset
, "UTF-8"))
505 /* Assume this for compatibility at present. */
511 typedef struct _GCharsetCache GCharsetCache
;
513 struct _GCharsetCache
{
520 charset_cache_free (gpointer data
)
522 GCharsetCache
*cache
= data
;
524 g_free (cache
->charset
);
530 * @charset: return location for character set name
532 * Obtains the character set for the <link linkend="setlocale">current
533 * locale</link>; you might use this character set as an argument to
534 * g_convert(), to convert from the current locale's encoding to some
535 * other encoding. (Frequently g_locale_to_utf8() and g_locale_from_utf8()
536 * are nice shortcuts, though.)
538 * On Windows the character set returned by this function is the
539 * so-called system default ANSI code-page. That is the character set
540 * used by the "narrow" versions of C library and Win32 functions that
541 * handle file names. It might be different from the character set
542 * used by the C library's current locale.
544 * The return value is %TRUE if the locale's encoding is UTF-8, in that
545 * case you can perhaps avoid calling g_convert().
547 * The string returned in @charset is not allocated, and should not be
550 * Return value: %TRUE if the returned charset is UTF-8
553 g_get_charset (G_CONST_RETURN
char **charset
)
555 static GStaticPrivate cache_private
= G_STATIC_PRIVATE_INIT
;
556 GCharsetCache
*cache
= g_static_private_get (&cache_private
);
561 cache
= g_new0 (GCharsetCache
, 1);
562 g_static_private_set (&cache_private
, cache
, charset_cache_free
);
565 raw
= _g_locale_charset_raw ();
567 if (!(cache
->raw
&& strcmp (cache
->raw
, raw
) == 0))
569 const gchar
*new_charset
;
572 g_free (cache
->charset
);
573 cache
->raw
= g_strdup (raw
);
574 cache
->is_utf8
= g_utf8_get_charset_internal (raw
, &new_charset
);
575 cache
->charset
= g_strdup (new_charset
);
579 *charset
= cache
->charset
;
581 return cache
->is_utf8
;
588 * @c: a Unicode character code
589 * @outbuf: output buffer, must have at least 6 bytes of space.
590 * If %NULL, the length will be computed and returned
591 * and nothing will be written to @outbuf.
593 * Converts a single character to UTF-8.
595 * Return value: number of bytes written
598 g_unichar_to_utf8 (gunichar c
,
601 /* If this gets modified, also update the copy in g_string_insert_unichar() */
616 else if (c
< 0x10000)
621 else if (c
< 0x200000)
626 else if (c
< 0x4000000)
639 for (i
= len
- 1; i
> 0; --i
)
641 outbuf
[i
] = (c
& 0x3f) | 0x80;
644 outbuf
[0] = c
| first
;
652 * @p: a nul-terminated UTF-8 encoded string
653 * @len: the maximum length of @p
654 * @c: a Unicode character
656 * Finds the leftmost occurrence of the given Unicode character
657 * in a UTF-8 encoded string, while limiting the search to @len bytes.
658 * If @len is -1, allow unbounded search.
660 * Return value: %NULL if the string does not contain the character,
661 * otherwise, a pointer to the start of the leftmost occurrence of
662 * the character in the string.
665 g_utf8_strchr (const char *p
,
671 gint charlen
= g_unichar_to_utf8 (c
, ch
);
674 return g_strstr_len (p
, len
, ch
);
680 * @p: a nul-terminated UTF-8 encoded string
681 * @len: the maximum length of @p
682 * @c: a Unicode character
684 * Find the rightmost occurrence of the given Unicode character
685 * in a UTF-8 encoded string, while limiting the search to @len bytes.
686 * If @len is -1, allow unbounded search.
688 * Return value: %NULL if the string does not contain the character,
689 * otherwise, a pointer to the start of the rightmost occurrence of the
690 * character in the string.
693 g_utf8_strrchr (const char *p
,
699 gint charlen
= g_unichar_to_utf8 (c
, ch
);
702 return g_strrstr_len (p
, len
, ch
);
706 /* Like g_utf8_get_char, but take a maximum length
707 * and return (gunichar)-2 on incomplete trailing character
709 static inline gunichar
710 g_utf8_get_char_extended (const gchar
*p
,
714 gunichar wc
= (guchar
) *p
;
754 if (max_len
>= 0 && len
> max_len
)
756 for (i
= 1; i
< max_len
; i
++)
758 if ((((guchar
*)p
)[i
] & 0xc0) != 0x80)
764 for (i
= 1; i
< len
; ++i
)
766 gunichar ch
= ((guchar
*)p
)[i
];
768 if ((ch
& 0xc0) != 0x80)
780 if (UTF8_LENGTH(wc
) != len
)
787 * g_utf8_get_char_validated:
788 * @p: a pointer to Unicode character encoded as UTF-8
789 * @max_len: the maximum number of bytes to read, or -1, for no maximum or
790 * if @p is nul-terminated
792 * Convert a sequence of bytes encoded as UTF-8 to a Unicode character.
793 * This function checks for incomplete characters, for invalid characters
794 * such as characters that are out of the range of Unicode, and for
795 * overlong encodings of valid characters.
797 * Return value: the resulting character. If @p points to a partial
798 * sequence at the end of a string that could begin a valid
799 * character (or if @max_len is zero), returns (gunichar)-2;
800 * otherwise, if @p does not point to a valid UTF-8 encoded
801 * Unicode character, returns (gunichar)-1.
804 g_utf8_get_char_validated (const gchar
*p
,
812 result
= g_utf8_get_char_extended (p
, max_len
);
814 if (result
& 0x80000000)
816 else if (!UNICODE_VALID (result
))
823 * g_utf8_to_ucs4_fast:
824 * @str: a UTF-8 encoded string
825 * @len: the maximum length of @str to use, in bytes. If @len < 0,
826 * then the string is nul-terminated.
827 * @items_written: location to store the number of characters in the
830 * Convert a string from UTF-8 to a 32-bit fixed width
831 * representation as UCS-4, assuming valid UTF-8 input.
832 * This function is roughly twice as fast as g_utf8_to_ucs4()
833 * but does no error checking on the input.
835 * Return value: a pointer to a newly allocated UCS-4 string.
836 * This value must be freed with g_free().
839 g_utf8_to_ucs4_fast (const gchar
*str
,
841 glong
*items_written
)
848 g_return_val_if_fail (str
!= NULL
, NULL
);
856 p
= g_utf8_next_char (p
);
862 while (p
< str
+ len
&& *p
)
864 p
= g_utf8_next_char (p
);
869 result
= g_new (gunichar
, n_chars
+ 1);
872 for (i
=0; i
< n_chars
; i
++)
874 gunichar wc
= ((unsigned char *)p
)[0];
909 for (j
= 1; j
< charlen
; j
++)
912 wc
|= ((unsigned char *)p
)[j
] & 0x3f;
929 * @str: a UTF-8 encoded string
930 * @len: the maximum length of @str to use, in bytes. If @len < 0,
931 * then the string is nul-terminated.
932 * @items_read: location to store number of bytes read, or %NULL.
933 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
934 * returned in case @str contains a trailing partial
935 * character. If an error occurs then the index of the
936 * invalid input is stored here.
937 * @items_written: location to store number of characters written or %NULL.
938 * The value here stored does not include the trailing 0
940 * @error: location to store the error occuring, or %NULL to ignore
941 * errors. Any of the errors in #GConvertError other than
942 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
944 * Convert a string from UTF-8 to a 32-bit fixed width
945 * representation as UCS-4. A trailing 0 will be added to the
946 * string after the converted text.
948 * Return value: a pointer to a newly allocated UCS-4 string.
949 * This value must be freed with g_free(). If an
950 * error occurs, %NULL will be returned and
954 g_utf8_to_ucs4 (const gchar
*str
,
957 glong
*items_written
,
960 gunichar
*result
= NULL
;
966 while ((len
< 0 || str
+ len
- in
> 0) && *in
)
968 gunichar wc
= g_utf8_get_char_extended (in
, len
< 0 ? 6 : str
+ len
- in
);
971 if (wc
== (gunichar
)-2)
976 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_PARTIAL_INPUT
,
977 _("Partial character sequence at end of input"));
980 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
981 _("Invalid byte sequence in conversion input"));
988 in
= g_utf8_next_char (in
);
991 result
= g_new (gunichar
, n_chars
+ 1);
994 for (i
=0; i
< n_chars
; i
++)
996 result
[i
] = g_utf8_get_char (in
);
997 in
= g_utf8_next_char (in
);
1002 *items_written
= n_chars
;
1006 *items_read
= in
- str
;
1013 * @str: a UCS-4 encoded string
1014 * @len: the maximum length (number of characters) of @str to use.
1015 * If @len < 0, then the string is nul-terminated.
1016 * @items_read: location to store number of characters read, or %NULL.
1017 * @items_written: location to store number of bytes written or %NULL.
1018 * The value here stored does not include the trailing 0
1020 * @error: location to store the error occuring, or %NULL to ignore
1021 * errors. Any of the errors in #GConvertError other than
1022 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1024 * Convert a string from a 32-bit fixed width representation as UCS-4.
1025 * to UTF-8. The result will be terminated with a 0 byte.
1027 * Return value: a pointer to a newly allocated UTF-8 string.
1028 * This value must be freed with g_free(). If an
1029 * error occurs, %NULL will be returned and
1030 * @error set. In that case, @items_read will be
1031 * set to the position of the first invalid input
1035 g_ucs4_to_utf8 (const gunichar
*str
,
1038 glong
*items_written
,
1042 gchar
*result
= NULL
;
1047 for (i
= 0; len
< 0 || i
< len
; i
++)
1052 if (str
[i
] >= 0x80000000)
1054 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1055 _("Character out of range for UTF-8"));
1059 result_length
+= UTF8_LENGTH (str
[i
]);
1062 result
= g_malloc (result_length
+ 1);
1066 while (p
< result
+ result_length
)
1067 p
+= g_unichar_to_utf8 (str
[i
++], p
);
1072 *items_written
= p
- result
;
1081 #define SURROGATE_VALUE(h,l) (((h) - 0xd800) * 0x400 + (l) - 0xdc00 + 0x10000)
1085 * @str: a UTF-16 encoded string
1086 * @len: the maximum length (number of <type>gunichar2</type>) of @str to use.
1087 * If @len < 0, then the string is nul-terminated.
1088 * @items_read: location to store number of words read, or %NULL.
1089 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1090 * returned in case @str contains a trailing partial
1091 * character. If an error occurs then the index of the
1092 * invalid input is stored here.
1093 * @items_written: location to store number of bytes written, or %NULL.
1094 * The value stored here does not include the trailing
1096 * @error: location to store the error occuring, or %NULL to ignore
1097 * errors. Any of the errors in #GConvertError other than
1098 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1100 * Convert a string from UTF-16 to UTF-8. The result will be
1101 * terminated with a 0 byte.
1103 * Note that the input is expected to be already in native endianness,
1104 * an initial byte-order-mark character is not handled specially.
1105 * g_convert() can be used to convert a byte buffer of UTF-16 data of
1106 * ambiguous endianess.
1108 * Further note that this function does not validate the result
1109 * string; it may e.g. include embedded NUL characters. The only
1110 * validation done by this function is to ensure that the input can
1111 * be correctly interpreted as UTF-16, i.e. it doesn't contain
1112 * things unpaired surrogates.
1114 * Return value: a pointer to a newly allocated UTF-8 string.
1115 * This value must be freed with g_free(). If an
1116 * error occurs, %NULL will be returned and
1120 g_utf16_to_utf8 (const gunichar2
*str
,
1123 glong
*items_written
,
1126 /* This function and g_utf16_to_ucs4 are almost exactly identical - The lines that differ
1129 const gunichar2
*in
;
1131 gchar
*result
= NULL
;
1133 gunichar high_surrogate
;
1135 g_return_val_if_fail (str
!= NULL
, NULL
);
1140 while ((len
< 0 || in
- str
< len
) && *in
)
1145 if (c
>= 0xdc00 && c
< 0xe000) /* low surrogate */
1149 wc
= SURROGATE_VALUE (high_surrogate
, c
);
1154 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1155 _("Invalid sequence in conversion input"));
1163 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1164 _("Invalid sequence in conversion input"));
1168 if (c
>= 0xd800 && c
< 0xdc00) /* high surrogate */
1177 /********** DIFFERENT for UTF8/UCS4 **********/
1178 n_bytes
+= UTF8_LENGTH (wc
);
1184 if (high_surrogate
&& !items_read
)
1186 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_PARTIAL_INPUT
,
1187 _("Partial character sequence at end of input"));
1191 /* At this point, everything is valid, and we just need to convert
1193 /********** DIFFERENT for UTF8/UCS4 **********/
1194 result
= g_malloc (n_bytes
+ 1);
1199 while (out
< result
+ n_bytes
)
1204 if (c
>= 0xdc00 && c
< 0xe000) /* low surrogate */
1206 wc
= SURROGATE_VALUE (high_surrogate
, c
);
1209 else if (c
>= 0xd800 && c
< 0xdc00) /* high surrogate */
1217 /********** DIFFERENT for UTF8/UCS4 **********/
1218 out
+= g_unichar_to_utf8 (wc
, out
);
1224 /********** DIFFERENT for UTF8/UCS4 **********/
1228 /********** DIFFERENT for UTF8/UCS4 **********/
1229 *items_written
= out
- result
;
1233 *items_read
= in
- str
;
1240 * @str: a UTF-16 encoded string
1241 * @len: the maximum length (number of <type>gunichar2</type>) of @str to use.
1242 * If @len < 0, then the string is nul-terminated.
1243 * @items_read: location to store number of words read, or %NULL.
1244 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1245 * returned in case @str contains a trailing partial
1246 * character. If an error occurs then the index of the
1247 * invalid input is stored here.
1248 * @items_written: location to store number of characters written, or %NULL.
1249 * The value stored here does not include the trailing
1251 * @error: location to store the error occuring, or %NULL to ignore
1252 * errors. Any of the errors in #GConvertError other than
1253 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1255 * Convert a string from UTF-16 to UCS-4. The result will be
1258 * Return value: a pointer to a newly allocated UCS-4 string.
1259 * This value must be freed with g_free(). If an
1260 * error occurs, %NULL will be returned and
1264 g_utf16_to_ucs4 (const gunichar2
*str
,
1267 glong
*items_written
,
1270 const gunichar2
*in
;
1272 gchar
*result
= NULL
;
1274 gunichar high_surrogate
;
1276 g_return_val_if_fail (str
!= NULL
, NULL
);
1281 while ((len
< 0 || in
- str
< len
) && *in
)
1286 if (c
>= 0xdc00 && c
< 0xe000) /* low surrogate */
1290 wc
= SURROGATE_VALUE (high_surrogate
, c
);
1295 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1296 _("Invalid sequence in conversion input"));
1304 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1305 _("Invalid sequence in conversion input"));
1309 if (c
>= 0xd800 && c
< 0xdc00) /* high surrogate */
1318 /********** DIFFERENT for UTF8/UCS4 **********/
1319 n_bytes
+= sizeof (gunichar
);
1325 if (high_surrogate
&& !items_read
)
1327 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_PARTIAL_INPUT
,
1328 _("Partial character sequence at end of input"));
1332 /* At this point, everything is valid, and we just need to convert
1334 /********** DIFFERENT for UTF8/UCS4 **********/
1335 result
= g_malloc (n_bytes
+ 4);
1340 while (out
< result
+ n_bytes
)
1345 if (c
>= 0xdc00 && c
< 0xe000) /* low surrogate */
1347 wc
= SURROGATE_VALUE (high_surrogate
, c
);
1350 else if (c
>= 0xd800 && c
< 0xdc00) /* high surrogate */
1358 /********** DIFFERENT for UTF8/UCS4 **********/
1359 *(gunichar
*)out
= wc
;
1360 out
+= sizeof (gunichar
);
1366 /********** DIFFERENT for UTF8/UCS4 **********/
1367 *(gunichar
*)out
= 0;
1370 /********** DIFFERENT for UTF8/UCS4 **********/
1371 *items_written
= (out
- result
) / sizeof (gunichar
);
1375 *items_read
= in
- str
;
1377 return (gunichar
*)result
;
1382 * @str: a UTF-8 encoded string
1383 * @len: the maximum length (number of bytes) of @str to use.
1384 * If @len < 0, then the string is nul-terminated.
1385 * @items_read: location to store number of bytes read, or %NULL.
1386 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1387 * returned in case @str contains a trailing partial
1388 * character. If an error occurs then the index of the
1389 * invalid input is stored here.
1390 * @items_written: location to store number of <type>gunichar2</type> written,
1392 * The value stored here does not include the trailing 0.
1393 * @error: location to store the error occuring, or %NULL to ignore
1394 * errors. Any of the errors in #GConvertError other than
1395 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1397 * Convert a string from UTF-8 to UTF-16. A 0 character will be
1398 * added to the result after the converted text.
1400 * Return value: a pointer to a newly allocated UTF-16 string.
1401 * This value must be freed with g_free(). If an
1402 * error occurs, %NULL will be returned and
1406 g_utf8_to_utf16 (const gchar
*str
,
1409 glong
*items_written
,
1412 gunichar2
*result
= NULL
;
1417 g_return_val_if_fail (str
!= NULL
, NULL
);
1421 while ((len
< 0 || str
+ len
- in
> 0) && *in
)
1423 gunichar wc
= g_utf8_get_char_extended (in
, len
< 0 ? 6 : str
+ len
- in
);
1424 if (wc
& 0x80000000)
1426 if (wc
== (gunichar
)-2)
1431 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_PARTIAL_INPUT
,
1432 _("Partial character sequence at end of input"));
1435 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1436 _("Invalid byte sequence in conversion input"));
1443 else if (wc
< 0xe000)
1445 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1446 _("Invalid sequence in conversion input"));
1450 else if (wc
< 0x10000)
1452 else if (wc
< 0x110000)
1456 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1457 _("Character out of range for UTF-16"));
1462 in
= g_utf8_next_char (in
);
1465 result
= g_new (gunichar2
, n16
+ 1);
1468 for (i
= 0; i
< n16
;)
1470 gunichar wc
= g_utf8_get_char (in
);
1478 result
[i
++] = (wc
- 0x10000) / 0x400 + 0xd800;
1479 result
[i
++] = (wc
- 0x10000) % 0x400 + 0xdc00;
1482 in
= g_utf8_next_char (in
);
1488 *items_written
= n16
;
1492 *items_read
= in
- str
;
1499 * @str: a UCS-4 encoded string
1500 * @len: the maximum length (number of characters) of @str to use.
1501 * If @len < 0, then the string is nul-terminated.
1502 * @items_read: location to store number of bytes read, or %NULL.
1503 * If an error occurs then the index of the invalid input
1505 * @items_written: location to store number of <type>gunichar2</type>
1506 * written, or %NULL. The value stored here does not
1507 * include the trailing 0.
1508 * @error: location to store the error occuring, or %NULL to ignore
1509 * errors. Any of the errors in #GConvertError other than
1510 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1512 * Convert a string from UCS-4 to UTF-16. A 0 character will be
1513 * added to the result after the converted text.
1515 * Return value: a pointer to a newly allocated UTF-16 string.
1516 * This value must be freed with g_free(). If an
1517 * error occurs, %NULL will be returned and
1521 g_ucs4_to_utf16 (const gunichar
*str
,
1524 glong
*items_written
,
1527 gunichar2
*result
= NULL
;
1533 while ((len
< 0 || i
< len
) && str
[i
])
1535 gunichar wc
= str
[i
];
1539 else if (wc
< 0xe000)
1541 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1542 _("Invalid sequence in conversion input"));
1546 else if (wc
< 0x10000)
1548 else if (wc
< 0x110000)
1552 g_set_error_literal (error
, G_CONVERT_ERROR
, G_CONVERT_ERROR_ILLEGAL_SEQUENCE
,
1553 _("Character out of range for UTF-16"));
1561 result
= g_new (gunichar2
, n16
+ 1);
1563 for (i
= 0, j
= 0; j
< n16
; i
++)
1565 gunichar wc
= str
[i
];
1573 result
[j
++] = (wc
- 0x10000) / 0x400 + 0xd800;
1574 result
[j
++] = (wc
- 0x10000) % 0x400 + 0xdc00;
1580 *items_written
= n16
;
1589 #define CONTINUATION_CHAR \
1591 if ((*(guchar *)p & 0xc0) != 0x80) /* 10xxxxxx */ \
1594 val |= (*(guchar *)p) & 0x3f; \
1597 static const gchar
*
1598 fast_validate (const char *str
)
1605 for (p
= str
; *p
; p
++)
1607 if (*(guchar
*)p
< 128)
1614 if ((*(guchar
*)p
& 0xe0) == 0xc0) /* 110xxxxx */
1616 if (G_UNLIKELY ((*(guchar
*)p
& 0x1e) == 0))
1619 if (G_UNLIKELY ((*(guchar
*)p
& 0xc0) != 0x80)) /* 10xxxxxx */
1624 if ((*(guchar
*)p
& 0xf0) == 0xe0) /* 1110xxxx */
1627 val
= *(guchar
*)p
& 0x0f;
1630 else if ((*(guchar
*)p
& 0xf8) == 0xf0) /* 11110xxx */
1633 val
= *(guchar
*)p
& 0x07;
1646 if (G_UNLIKELY (val
< min
))
1649 if (G_UNLIKELY (!UNICODE_VALID(val
)))
1663 static const gchar
*
1664 fast_validate_len (const char *str
,
1672 g_assert (max_len
>= 0);
1674 for (p
= str
; ((p
- str
) < max_len
) && *p
; p
++)
1676 if (*(guchar
*)p
< 128)
1683 if ((*(guchar
*)p
& 0xe0) == 0xc0) /* 110xxxxx */
1685 if (G_UNLIKELY (max_len
- (p
- str
) < 2))
1688 if (G_UNLIKELY ((*(guchar
*)p
& 0x1e) == 0))
1691 if (G_UNLIKELY ((*(guchar
*)p
& 0xc0) != 0x80)) /* 10xxxxxx */
1696 if ((*(guchar
*)p
& 0xf0) == 0xe0) /* 1110xxxx */
1698 if (G_UNLIKELY (max_len
- (p
- str
) < 3))
1702 val
= *(guchar
*)p
& 0x0f;
1705 else if ((*(guchar
*)p
& 0xf8) == 0xf0) /* 11110xxx */
1707 if (G_UNLIKELY (max_len
- (p
- str
) < 4))
1711 val
= *(guchar
*)p
& 0x07;
1724 if (G_UNLIKELY (val
< min
))
1726 if (G_UNLIKELY (!UNICODE_VALID(val
)))
1742 * @str: a pointer to character data
1743 * @max_len: max bytes to validate, or -1 to go until NUL
1744 * @end: return location for end of valid data
1746 * Validates UTF-8 encoded text. @str is the text to validate;
1747 * if @str is nul-terminated, then @max_len can be -1, otherwise
1748 * @max_len should be the number of bytes to validate.
1749 * If @end is non-%NULL, then the end of the valid range
1750 * will be stored there (i.e. the start of the first invalid
1751 * character if some bytes were invalid, or the end of the text
1752 * being validated otherwise).
1754 * Note that g_utf8_validate() returns %FALSE if @max_len is
1755 * positive and NUL is met before @max_len bytes have been read.
1757 * Returns %TRUE if all of @str was valid. Many GLib and GTK+
1758 * routines <emphasis>require</emphasis> valid UTF-8 as input;
1759 * so data read from a file or the network should be checked
1760 * with g_utf8_validate() before doing anything else with it.
1762 * Return value: %TRUE if the text was valid UTF-8
1765 g_utf8_validate (const char *str
,
1773 p
= fast_validate (str
);
1775 p
= fast_validate_len (str
, max_len
);
1780 if ((max_len
>= 0 && p
!= str
+ max_len
) ||
1781 (max_len
< 0 && *p
!= '\0'))
1788 * g_unichar_validate:
1789 * @ch: a Unicode character
1791 * Checks whether @ch is a valid Unicode character. Some possible
1792 * integer values of @ch will not be valid. 0 is considered a valid
1793 * character, though it's normally a string terminator.
1795 * Return value: %TRUE if @ch is a valid Unicode character
1798 g_unichar_validate (gunichar ch
)
1800 return UNICODE_VALID (ch
);
1804 * g_utf8_strreverse:
1805 * @str: a UTF-8 encoded string
1806 * @len: the maximum length of @str to use, in bytes. If @len < 0,
1807 * then the string is nul-terminated.
1809 * Reverses a UTF-8 string. @str must be valid UTF-8 encoded text.
1810 * (Use g_utf8_validate() on all text before trying to use UTF-8
1811 * utility functions with it.)
1813 * This function is intended for programmatic uses of reversed strings.
1814 * It pays no attention to decomposed characters, combining marks, byte
1815 * order marks, directional indicators (LRM, LRO, etc) and similar
1816 * characters which might need special handling when reversing a string
1817 * for display purposes.
1819 * Note that unlike g_strreverse(), this function returns
1820 * newly-allocated memory, which should be freed with g_free() when
1823 * Returns: a newly-allocated string which is the reverse of @str.
1828 g_utf8_strreverse (const gchar
*str
,
1837 result
= g_new (gchar
, len
+ 1);
1842 gchar
*m
, skip
= g_utf8_skip
[*(guchar
*) p
];
1844 for (m
= r
; skip
; skip
--)
1854 _g_utf8_make_valid (const gchar
*name
)
1857 const gchar
*remainder
, *invalid
;
1858 gint remaining_bytes
, valid_bytes
;
1860 g_return_val_if_fail (name
!= NULL
, NULL
);
1864 remaining_bytes
= strlen (name
);
1866 while (remaining_bytes
!= 0)
1868 if (g_utf8_validate (remainder
, remaining_bytes
, &invalid
))
1870 valid_bytes
= invalid
- remainder
;
1873 string
= g_string_sized_new (remaining_bytes
);
1875 g_string_append_len (string
, remainder
, valid_bytes
);
1876 /* append U+FFFD REPLACEMENT CHARACTER */
1877 g_string_append (string
, "\357\277\275");
1879 remaining_bytes
-= valid_bytes
+ 1;
1880 remainder
= invalid
+ 1;
1884 return g_strdup (name
);
1886 g_string_append (string
, remainder
);
1888 g_assert (g_utf8_validate (string
->str
, -1, NULL
));
1890 return g_string_free (string
, FALSE
);
1894 #define __G_UTF8_C__
1895 #include "galiasdef.c"