2 * Copyright (C) 2011, 2012 Apple Inc. All rights reserved.
3 * Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public License
16 * along with this library; see the file COPYING.LIB. If not, write to
17 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
22 #ifndef ASCIIFastPath_h
23 #define ASCIIFastPath_h
25 #include "wtf/Alignment.h"
27 #include "wtf/StdLibExtras.h"
28 #include "wtf/text/Unicode.h"
31 #if OS(MACOSX) && (CPU(X86) || CPU(X86_64))
32 #include <emmintrin.h>
37 // Assuming that a pointer is the size of a "machine word", then
38 // uintptr_t is an integer type that is also a machine word.
39 typedef uintptr_t MachineWord
;
40 const uintptr_t machineWordAlignmentMask
= sizeof(MachineWord
) - 1;
42 inline bool isAlignedToMachineWord(const void* pointer
)
44 return !(reinterpret_cast<uintptr_t>(pointer
) & machineWordAlignmentMask
);
47 template<typename T
> inline T
* alignToMachineWord(T
* pointer
)
49 return reinterpret_cast<T
*>(reinterpret_cast<uintptr_t>(pointer
) & ~machineWordAlignmentMask
);
52 template<size_t size
, typename CharacterType
> struct NonASCIIMask
;
53 template<> struct NonASCIIMask
<4, UChar
> {
54 static inline uint32_t value() { return 0xFF80FF80U
; }
56 template<> struct NonASCIIMask
<4, LChar
> {
57 static inline uint32_t value() { return 0x80808080U
; }
59 template<> struct NonASCIIMask
<8, UChar
> {
60 static inline uint64_t value() { return 0xFF80FF80FF80FF80ULL
; }
62 template<> struct NonASCIIMask
<8, LChar
> {
63 static inline uint64_t value() { return 0x8080808080808080ULL
; }
67 template<typename CharacterType
>
68 inline bool isAllASCII(MachineWord word
)
70 return !(word
& NonASCIIMask
<sizeof(MachineWord
), CharacterType
>::value());
73 // Note: This function assume the input is likely all ASCII, and
74 // does not leave early if it is not the case.
75 template<typename CharacterType
>
76 inline bool charactersAreAllASCII(const CharacterType
* characters
, size_t length
)
78 MachineWord allCharBits
= 0;
79 const CharacterType
* end
= characters
+ length
;
81 // Prologue: align the input.
82 while (!isAlignedToMachineWord(characters
) && characters
!= end
) {
83 allCharBits
|= *characters
;
87 // Compare the values of CPU word size.
88 const CharacterType
* wordEnd
= alignToMachineWord(end
);
89 const size_t loopIncrement
= sizeof(MachineWord
) / sizeof(CharacterType
);
90 while (characters
< wordEnd
) {
91 allCharBits
|= *(reinterpret_cast_ptr
<const MachineWord
*>(characters
));
92 characters
+= loopIncrement
;
95 // Process the remaining bytes.
96 while (characters
!= end
) {
97 allCharBits
|= *characters
;
101 MachineWord nonASCIIBitMask
= NonASCIIMask
<sizeof(MachineWord
), CharacterType
>::value();
102 return !(allCharBits
& nonASCIIBitMask
);
105 inline void copyLCharsFromUCharSource(LChar
* destination
, const UChar
* source
, size_t length
)
107 #if OS(MACOSX) && (CPU(X86) || CPU(X86_64))
108 const uintptr_t memoryAccessSize
= 16; // Memory accesses on 16 byte (128 bit) alignment
109 const uintptr_t memoryAccessMask
= memoryAccessSize
- 1;
112 for (;i
< length
&& !isAlignedTo
<memoryAccessMask
>(&source
[i
]); ++i
) {
113 ASSERT(!(source
[i
] & 0xff00));
114 destination
[i
] = static_cast<LChar
>(source
[i
]);
117 const uintptr_t sourceLoadSize
= 32; // Process 32 bytes (16 UChars) each iteration
118 const size_t ucharsPerLoop
= sourceLoadSize
/ sizeof(UChar
);
119 if (length
> ucharsPerLoop
) {
120 const size_t endLength
= length
- ucharsPerLoop
+ 1;
121 for (; i
< endLength
; i
+= ucharsPerLoop
) {
123 for (unsigned checkIndex
= 0; checkIndex
< ucharsPerLoop
; ++checkIndex
)
124 ASSERT(!(source
[i
+checkIndex
] & 0xff00));
126 __m128i first8UChars
= _mm_load_si128(reinterpret_cast<const __m128i
*>(&source
[i
]));
127 __m128i second8UChars
= _mm_load_si128(reinterpret_cast<const __m128i
*>(&source
[i
+8]));
128 __m128i packedChars
= _mm_packus_epi16(first8UChars
, second8UChars
);
129 _mm_storeu_si128(reinterpret_cast<__m128i
*>(&destination
[i
]), packedChars
);
133 for (; i
< length
; ++i
) {
134 ASSERT(!(source
[i
] & 0xff00));
135 destination
[i
] = static_cast<LChar
>(source
[i
]);
137 #elif COMPILER(GCC) && CPU(ARM_NEON) && !(CPU(BIG_ENDIAN) || CPU(MIDDLE_ENDIAN)) && defined(NDEBUG)
138 const LChar
* const end
= destination
+ length
;
139 const uintptr_t memoryAccessSize
= 8;
141 if (length
>= (2 * memoryAccessSize
) - 1) {
142 // Prefix: align dst on 64 bits.
143 const uintptr_t memoryAccessMask
= memoryAccessSize
- 1;
144 while (!isAlignedTo
<memoryAccessMask
>(destination
))
145 *destination
++ = static_cast<LChar
>(*source
++);
147 // Vector interleaved unpack, we only store the lower 8 bits.
148 const uintptr_t lengthLeft
= end
- destination
;
149 const LChar
* const simdEnd
= end
- (lengthLeft
% memoryAccessSize
);
151 asm("vld2.8 { d0-d1 }, [%[SOURCE]] !\n\t"
152 "vst1.8 { d0 }, [%[DESTINATION],:64] !\n\t"
153 : [SOURCE
]"+r" (source
), [DESTINATION
]"+r" (destination
)
155 : "memory", "d0", "d1");
156 } while (destination
!= simdEnd
);
159 while (destination
!= end
)
160 *destination
++ = static_cast<LChar
>(*source
++);
162 for (size_t i
= 0; i
< length
; ++i
) {
163 ASSERT(!(source
[i
] & 0xff00));
164 destination
[i
] = static_cast<LChar
>(source
[i
]);
171 #endif // ASCIIFastPath_h