2 * csum_partial_copy - do IP checksumming and copy
4 * (C) Copyright 1996 Linus Torvalds
5 * accelerated versions (and 21264 assembly versions ) contributed by
6 * Rick Gorton <rick.gorton@alpha-processor.com>
8 * Don't look at this too closely - you'll go mad. The things
9 * we do for performance..
12 #include <linux/types.h>
13 #include <linux/string.h>
14 #include <asm/uaccess.h>
18 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
21 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
23 #define extql(x,y,z) \
24 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
26 #define extqh(x,y,z) \
27 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
29 #define mskql(x,y,z) \
30 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
32 #define mskqh(x,y,z) \
33 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
35 #define insql(x,y,z) \
36 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
38 #define insqh(x,y,z) \
39 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
42 #define __get_user_u(x,ptr) \
45 __asm__ __volatile__( \
48 ".section __ex_table,\"a\"\n" \
50 " lda %0,2b-1b(%1)\n" \
52 : "=r"(x), "=r"(__guu_err) \
53 : "m"(__m(ptr)), "1"(0)); \
57 #define __put_user_u(x,ptr) \
60 __asm__ __volatile__( \
63 ".section __ex_table,\"a\"\n" \
65 " lda $31,2b-1b(%0)\n" \
68 : "m"(__m(addr)), "rJ"(x), "0"(0)); \
73 static inline unsigned short from64to16(unsigned long x
)
75 /* Using extract instructions is a bit more efficient
76 than the original shift/bitmask version. */
85 tmp_v
.ul
= (unsigned long) in_v
.ui
[0] + (unsigned long) in_v
.ui
[1];
87 /* Since the bits of tmp_v.sh[3] are going to always be zero,
88 we don't have to bother to add that in. */
89 out_v
.ul
= (unsigned long) tmp_v
.us
[0] + (unsigned long) tmp_v
.us
[1]
90 + (unsigned long) tmp_v
.us
[2];
92 /* Similarly, out_v.us[2] is always zero for the final add. */
93 return out_v
.us
[0] + out_v
.us
[1];
99 * Ok. This isn't fun, but this is the EASY case.
101 static inline unsigned long
102 csum_partial_cfu_aligned(const unsigned long __user
*src
, unsigned long *dst
,
103 long len
, unsigned long checksum
,
106 unsigned long carry
= 0;
111 err
|= __get_user(word
, src
);
116 carry
= checksum
< word
;
123 unsigned long word
, tmp
;
124 err
|= __get_user(word
, src
);
126 mskql(word
, len
, word
);
128 mskqh(tmp
, len
, tmp
);
129 carry
= checksum
< word
;
133 if (err
) *errp
= err
;
138 * This is even less fun, but this is still reasonably
141 static inline unsigned long
142 csum_partial_cfu_dest_aligned(const unsigned long __user
*src
,
145 long len
, unsigned long checksum
,
149 unsigned long word
, carry
;
150 unsigned long lastsrc
= 7+len
+(unsigned long)src
;
153 err
|= __get_user_u(first
,src
);
156 unsigned long second
;
158 err
|= __get_user_u(second
, src
+1);
159 extql(first
, soff
, word
);
162 extqh(second
, soff
, first
);
169 carry
= checksum
< word
;
175 unsigned long second
;
176 err
|= __get_user_u(second
, lastsrc
);
178 extql(first
, soff
, word
);
179 extqh(second
, soff
, first
);
181 mskql(word
, len
, word
);
183 mskqh(tmp
, len
, tmp
);
184 carry
= checksum
< word
;
188 if (err
) *errp
= err
;
193 * This is slightly less fun than the above..
195 static inline unsigned long
196 csum_partial_cfu_src_aligned(const unsigned long __user
*src
,
199 long len
, unsigned long checksum
,
200 unsigned long partial_dest
,
203 unsigned long carry
= 0;
205 unsigned long second_dest
;
208 mskql(partial_dest
, doff
, partial_dest
);
210 err
|= __get_user(word
, src
);
212 insql(word
, doff
, second_dest
);
214 stq_u(partial_dest
| second_dest
, dst
);
217 insqh(word
, doff
, partial_dest
);
218 carry
= checksum
< word
;
224 err
|= __get_user(word
, src
);
225 mskql(word
, len
, word
);
228 insql(word
, doff
, second_dest
);
230 carry
= checksum
< word
;
231 partial_dest
|= second_dest
;
233 stq_u(partial_dest
, dst
);
236 insqh(word
, doff
, partial_dest
);
240 ldq_u(second_dest
, dst
);
241 mskqh(second_dest
, doff
, second_dest
);
242 stq_u(partial_dest
| second_dest
, dst
);
245 if (err
) *errp
= err
;
250 * This is so totally un-fun that it's frightening. Don't
251 * look at this too closely, you'll go blind.
253 static inline unsigned long
254 csum_partial_cfu_unaligned(const unsigned long __user
* src
,
256 unsigned long soff
, unsigned long doff
,
257 long len
, unsigned long checksum
,
258 unsigned long partial_dest
,
261 unsigned long carry
= 0;
263 unsigned long lastsrc
;
266 err
|= __get_user_u(first
, src
);
267 lastsrc
= 7+len
+(unsigned long)src
;
268 mskql(partial_dest
, doff
, partial_dest
);
270 unsigned long second
, word
;
271 unsigned long second_dest
;
273 err
|= __get_user_u(second
, src
+1);
274 extql(first
, soff
, word
);
277 extqh(second
, soff
, first
);
281 insql(word
, doff
, second_dest
);
283 stq_u(partial_dest
| second_dest
, dst
);
284 carry
= checksum
< word
;
285 insqh(word
, doff
, partial_dest
);
291 unsigned long second
, word
;
292 unsigned long second_dest
;
294 err
|= __get_user_u(second
, lastsrc
);
295 extql(first
, soff
, word
);
296 extqh(second
, soff
, first
);
299 mskql(word
, len
-doff
, word
);
301 insql(word
, doff
, second_dest
);
302 carry
= checksum
< word
;
303 stq_u(partial_dest
| second_dest
, dst
);
305 ldq_u(second_dest
, dst
+1);
306 insqh(word
, doff
, partial_dest
);
307 mskqh(second_dest
, len
, second_dest
);
308 stq_u(partial_dest
| second_dest
, dst
+1);
312 unsigned long second
, word
;
313 unsigned long second_dest
;
315 err
|= __get_user_u(second
, lastsrc
);
316 extql(first
, soff
, word
);
317 extqh(second
, soff
, first
);
319 ldq_u(second_dest
, dst
);
320 mskql(word
, len
-doff
, word
);
322 mskqh(second_dest
, len
, second_dest
);
323 carry
= checksum
< word
;
324 insql(word
, doff
, word
);
325 stq_u(partial_dest
| word
| second_dest
, dst
);
328 if (err
) *errp
= err
;
333 csum_partial_copy_from_user(const void __user
*src
, void *dst
, int len
,
334 __wsum sum
, int *errp
)
336 unsigned long checksum
= (__force u32
) sum
;
337 unsigned long soff
= 7 & (unsigned long) src
;
338 unsigned long doff
= 7 & (unsigned long) dst
;
341 if (!access_ok(VERIFY_READ
, src
, len
)) {
348 checksum
= csum_partial_cfu_aligned(
349 (const unsigned long __user
*) src
,
350 (unsigned long *) dst
,
351 len
-8, checksum
, errp
);
353 checksum
= csum_partial_cfu_dest_aligned(
354 (const unsigned long __user
*) src
,
355 (unsigned long *) dst
,
356 soff
, len
-8, checksum
, errp
);
358 unsigned long partial_dest
;
359 ldq_u(partial_dest
, dst
);
361 checksum
= csum_partial_cfu_src_aligned(
362 (const unsigned long __user
*) src
,
363 (unsigned long *) dst
,
364 doff
, len
-8, checksum
,
367 checksum
= csum_partial_cfu_unaligned(
368 (const unsigned long __user
*) src
,
369 (unsigned long *) dst
,
370 soff
, doff
, len
-8, checksum
,
373 checksum
= from64to16 (checksum
);
375 return (__force __wsum
)checksum
;
379 csum_partial_copy_nocheck(const void *src
, void *dst
, int len
, __wsum sum
)
382 mm_segment_t oldfs
= get_fs();
384 checksum
= csum_partial_copy_from_user((__force
const void __user
*)src
,
385 dst
, len
, sum
, NULL
);