1 // SPDX-License-Identifier: GPL-2.0
3 * csum_partial_copy - do IP checksumming and copy
5 * (C) Copyright 1996 Linus Torvalds
6 * accelerated versions (and 21264 assembly versions ) contributed by
7 * Rick Gorton <rick.gorton@alpha-processor.com>
9 * Don't look at this too closely - you'll go mad. The things
10 * we do for performance..
13 #include <linux/types.h>
14 #include <linux/string.h>
15 #include <linux/uaccess.h>
19 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
22 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
24 #define extql(x,y,z) \
25 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
27 #define extqh(x,y,z) \
28 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
30 #define mskql(x,y,z) \
31 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
33 #define mskqh(x,y,z) \
34 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
36 #define insql(x,y,z) \
37 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
39 #define insqh(x,y,z) \
40 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
42 #define __get_word(insn,x,ptr) \
45 __asm__ __volatile__( \
46 "1: "#insn" %0,%2\n" \
49 : "=r"(x), "=r"(__guu_err) \
50 : "m"(__m(ptr)), "1"(0)); \
54 static inline unsigned short from64to16(unsigned long x
)
56 /* Using extract instructions is a bit more efficient
57 than the original shift/bitmask version. */
66 tmp_v
.ul
= (unsigned long) in_v
.ui
[0] + (unsigned long) in_v
.ui
[1];
68 /* Since the bits of tmp_v.sh[3] are going to always be zero,
69 we don't have to bother to add that in. */
70 out_v
.ul
= (unsigned long) tmp_v
.us
[0] + (unsigned long) tmp_v
.us
[1]
71 + (unsigned long) tmp_v
.us
[2];
73 /* Similarly, out_v.us[2] is always zero for the final add. */
74 return out_v
.us
[0] + out_v
.us
[1];
80 * Ok. This isn't fun, but this is the EASY case.
82 static inline unsigned long
83 csum_partial_cfu_aligned(const unsigned long __user
*src
, unsigned long *dst
,
86 unsigned long checksum
= ~0U;
87 unsigned long carry
= 0;
91 if (__get_word(ldq
, word
, src
))
97 carry
= checksum
< word
;
104 unsigned long word
, tmp
;
105 if (__get_word(ldq
, word
, src
))
108 mskql(word
, len
, word
);
110 mskqh(tmp
, len
, tmp
);
111 carry
= checksum
< word
;
119 * This is even less fun, but this is still reasonably
122 static inline unsigned long
123 csum_partial_cfu_dest_aligned(const unsigned long __user
*src
,
129 unsigned long word
, carry
;
130 unsigned long lastsrc
= 7+len
+(unsigned long)src
;
131 unsigned long checksum
= ~0U;
133 if (__get_word(ldq_u
, first
,src
))
137 unsigned long second
;
139 if (__get_word(ldq_u
, second
, src
+1))
141 extql(first
, soff
, word
);
144 extqh(second
, soff
, first
);
151 carry
= checksum
< word
;
157 unsigned long second
;
158 if (__get_word(ldq_u
, second
, lastsrc
))
161 extql(first
, soff
, word
);
162 extqh(second
, soff
, first
);
164 mskql(word
, len
, word
);
166 mskqh(tmp
, len
, tmp
);
167 carry
= checksum
< word
;
175 * This is slightly less fun than the above..
177 static inline unsigned long
178 csum_partial_cfu_src_aligned(const unsigned long __user
*src
,
182 unsigned long partial_dest
)
184 unsigned long carry
= 0;
186 unsigned long second_dest
;
187 unsigned long checksum
= ~0U;
189 mskql(partial_dest
, doff
, partial_dest
);
191 if (__get_word(ldq
, word
, src
))
194 insql(word
, doff
, second_dest
);
196 stq_u(partial_dest
| second_dest
, dst
);
199 insqh(word
, doff
, partial_dest
);
200 carry
= checksum
< word
;
206 if (__get_word(ldq
, word
, src
))
208 mskql(word
, len
, word
);
211 insql(word
, doff
, second_dest
);
213 carry
= checksum
< word
;
214 partial_dest
|= second_dest
;
216 stq_u(partial_dest
, dst
);
219 insqh(word
, doff
, partial_dest
);
223 ldq_u(second_dest
, dst
);
224 mskqh(second_dest
, doff
, second_dest
);
225 stq_u(partial_dest
| second_dest
, dst
);
232 * This is so totally un-fun that it's frightening. Don't
233 * look at this too closely, you'll go blind.
235 static inline unsigned long
236 csum_partial_cfu_unaligned(const unsigned long __user
* src
,
238 unsigned long soff
, unsigned long doff
,
239 long len
, unsigned long partial_dest
)
241 unsigned long carry
= 0;
243 unsigned long lastsrc
;
244 unsigned long checksum
= ~0U;
246 if (__get_word(ldq_u
, first
, src
))
248 lastsrc
= 7+len
+(unsigned long)src
;
249 mskql(partial_dest
, doff
, partial_dest
);
251 unsigned long second
, word
;
252 unsigned long second_dest
;
254 if (__get_word(ldq_u
, second
, src
+1))
256 extql(first
, soff
, word
);
259 extqh(second
, soff
, first
);
263 insql(word
, doff
, second_dest
);
265 stq_u(partial_dest
| second_dest
, dst
);
266 carry
= checksum
< word
;
267 insqh(word
, doff
, partial_dest
);
273 unsigned long second
, word
;
274 unsigned long second_dest
;
276 if (__get_word(ldq_u
, second
, lastsrc
))
278 extql(first
, soff
, word
);
279 extqh(second
, soff
, first
);
282 mskql(word
, len
-doff
, word
);
284 insql(word
, doff
, second_dest
);
285 carry
= checksum
< word
;
286 stq_u(partial_dest
| second_dest
, dst
);
288 ldq_u(second_dest
, dst
+1);
289 insqh(word
, doff
, partial_dest
);
290 mskqh(second_dest
, len
, second_dest
);
291 stq_u(partial_dest
| second_dest
, dst
+1);
295 unsigned long second
, word
;
296 unsigned long second_dest
;
298 if (__get_word(ldq_u
, second
, lastsrc
))
300 extql(first
, soff
, word
);
301 extqh(second
, soff
, first
);
303 ldq_u(second_dest
, dst
);
304 mskql(word
, len
-doff
, word
);
306 mskqh(second_dest
, len
, second_dest
);
307 carry
= checksum
< word
;
308 insql(word
, doff
, word
);
309 stq_u(partial_dest
| word
| second_dest
, dst
);
315 static __wsum
__csum_and_copy(const void __user
*src
, void *dst
, int len
)
317 unsigned long soff
= 7 & (unsigned long) src
;
318 unsigned long doff
= 7 & (unsigned long) dst
;
319 unsigned long checksum
;
323 checksum
= csum_partial_cfu_aligned(
324 (const unsigned long __user
*) src
,
325 (unsigned long *) dst
, len
-8);
327 checksum
= csum_partial_cfu_dest_aligned(
328 (const unsigned long __user
*) src
,
329 (unsigned long *) dst
,
332 unsigned long partial_dest
;
333 ldq_u(partial_dest
, dst
);
335 checksum
= csum_partial_cfu_src_aligned(
336 (const unsigned long __user
*) src
,
337 (unsigned long *) dst
,
338 doff
, len
-8, partial_dest
);
340 checksum
= csum_partial_cfu_unaligned(
341 (const unsigned long __user
*) src
,
342 (unsigned long *) dst
,
343 soff
, doff
, len
-8, partial_dest
);
345 return (__force __wsum
)from64to16 (checksum
);
349 csum_and_copy_from_user(const void __user
*src
, void *dst
, int len
)
351 if (!access_ok(src
, len
))
353 return __csum_and_copy(src
, dst
, len
);
355 EXPORT_SYMBOL(csum_and_copy_from_user
);
358 csum_partial_copy_nocheck(const void *src
, void *dst
, int len
)
360 return __csum_and_copy((__force
const void __user
*)src
,
363 EXPORT_SYMBOL(csum_partial_copy_nocheck
);