2 * linux/arch/arm/lib/uaccess.S
4 * Copyright (C) 1995, 1996,1997,1998 Russell King
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * Routines to block copy data to/from user memory
11 * These are highly optimised both for the 4k page size
12 * and for various alignments.
14 #include <linux/linkage.h>
15 #include <asm/assembler.h>
16 #include <asm/errno.h>
17 #include <asm/domain.h>
23 /* Prototype: int __copy_to_user(void *to, const char *from, size_t n)
24 * Purpose : copy a block to user memory from kernel memory
25 * Params : to - user memory
26 * : from - kernel memory
27 * : n - number of bytes to copy
28 * Returns : Number of bytes NOT copied.
31 .Lc2u_dest_not_aligned:
35 USER( TUSER( strb) r3, [r0], #1) @ May fault
37 USER( TUSER( strgeb) r3, [r0], #1) @ May fault
39 USER( TUSER( strgtb) r3, [r0], #1) @ May fault
44 stmfd sp!, {r2, r4 - r7, lr}
48 bne .Lc2u_dest_not_aligned
52 bne .Lc2u_src_not_aligned
54 * Seeing as there has to be at least 8 bytes to copy, we can
55 * copy one word, and force a user-mode page fault...
58 .Lc2u_0fupi: subs r2, r2, #4
62 USER( TUSER( str) r3, [r0], #4) @ May fault
63 mov ip, r0, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
65 movs ip, ip, lsr #32 - PAGE_SHIFT
68 * ip = max no. of bytes to copy before needing another "strt" insn
76 .Lc2u_0cpy8lp: ldmia r1!, {r3 - r6}
77 stmia r0!, {r3 - r6} @ Shouldnt fault
80 stmia r0!, {r3 - r6} @ Shouldnt fault
83 .Lc2u_0rem8lp: cmn ip, #16
84 ldmgeia r1!, {r3 - r6}
85 stmgeia r0!, {r3 - r6} @ Shouldnt fault
87 ldmneia r1!, {r3 - r4}
88 stmneia r0!, {r3 - r4} @ Shouldnt fault
91 TUSER( strne) r3, [r0], #4 @ Shouldnt fault
94 .Lc2u_0nowords: teq ip, #0
96 .Lc2u_nowords: cmp ip, #2
98 USER( TUSER( strb) r3, [r0], #1) @ May fault
100 USER( TUSER( strgeb) r3, [r0], #1) @ May fault
102 USER( TUSER( strgtb) r3, [r0], #1) @ May fault
108 .Lc2u_finished: mov r0, #0
109 ldmfd sp!, {r2, r4 - r7, pc}
111 .Lc2u_src_not_aligned:
117 .Lc2u_1fupi: subs r2, r2, #4
120 mov r3, r7, lspull #8
122 orr r3, r3, r7, lspush #24
123 USER( TUSER( str) r3, [r0], #4) @ May fault
124 mov ip, r0, lsl #32 - PAGE_SHIFT
126 movs ip, ip, lsr #32 - PAGE_SHIFT
134 .Lc2u_1cpy8lp: mov r3, r7, lspull #8
137 orr r3, r3, r4, lspush #24
138 mov r4, r4, lspull #8
139 orr r4, r4, r5, lspush #24
140 mov r5, r5, lspull #8
141 orr r5, r5, r6, lspush #24
142 mov r6, r6, lspull #8
143 orr r6, r6, r7, lspush #24
144 stmia r0!, {r3 - r6} @ Shouldnt fault
147 .Lc2u_1rem8lp: tst ip, #8
148 movne r3, r7, lspull #8
149 ldmneia r1!, {r4, r7}
150 orrne r3, r3, r4, lspush #24
151 movne r4, r4, lspull #8
152 orrne r4, r4, r7, lspush #24
153 stmneia r0!, {r3 - r4} @ Shouldnt fault
155 movne r3, r7, lspull #8
157 orrne r3, r3, r7, lspush #24
158 TUSER( strne) r3, [r0], #4 @ Shouldnt fault
161 .Lc2u_1nowords: mov r3, r7, get_byte_1
165 USER( TUSER( strb) r3, [r0], #1) @ May fault
166 movge r3, r7, get_byte_2
167 USER( TUSER( strgeb) r3, [r0], #1) @ May fault
168 movgt r3, r7, get_byte_3
169 USER( TUSER( strgtb) r3, [r0], #1) @ May fault
172 .Lc2u_2fupi: subs r2, r2, #4
175 mov r3, r7, lspull #16
177 orr r3, r3, r7, lspush #16
178 USER( TUSER( str) r3, [r0], #4) @ May fault
179 mov ip, r0, lsl #32 - PAGE_SHIFT
181 movs ip, ip, lsr #32 - PAGE_SHIFT
189 .Lc2u_2cpy8lp: mov r3, r7, lspull #16
192 orr r3, r3, r4, lspush #16
193 mov r4, r4, lspull #16
194 orr r4, r4, r5, lspush #16
195 mov r5, r5, lspull #16
196 orr r5, r5, r6, lspush #16
197 mov r6, r6, lspull #16
198 orr r6, r6, r7, lspush #16
199 stmia r0!, {r3 - r6} @ Shouldnt fault
202 .Lc2u_2rem8lp: tst ip, #8
203 movne r3, r7, lspull #16
204 ldmneia r1!, {r4, r7}
205 orrne r3, r3, r4, lspush #16
206 movne r4, r4, lspull #16
207 orrne r4, r4, r7, lspush #16
208 stmneia r0!, {r3 - r4} @ Shouldnt fault
210 movne r3, r7, lspull #16
212 orrne r3, r3, r7, lspush #16
213 TUSER( strne) r3, [r0], #4 @ Shouldnt fault
216 .Lc2u_2nowords: mov r3, r7, get_byte_2
220 USER( TUSER( strb) r3, [r0], #1) @ May fault
221 movge r3, r7, get_byte_3
222 USER( TUSER( strgeb) r3, [r0], #1) @ May fault
224 USER( TUSER( strgtb) r3, [r0], #1) @ May fault
227 .Lc2u_3fupi: subs r2, r2, #4
230 mov r3, r7, lspull #24
232 orr r3, r3, r7, lspush #8
233 USER( TUSER( str) r3, [r0], #4) @ May fault
234 mov ip, r0, lsl #32 - PAGE_SHIFT
236 movs ip, ip, lsr #32 - PAGE_SHIFT
244 .Lc2u_3cpy8lp: mov r3, r7, lspull #24
247 orr r3, r3, r4, lspush #8
248 mov r4, r4, lspull #24
249 orr r4, r4, r5, lspush #8
250 mov r5, r5, lspull #24
251 orr r5, r5, r6, lspush #8
252 mov r6, r6, lspull #24
253 orr r6, r6, r7, lspush #8
254 stmia r0!, {r3 - r6} @ Shouldnt fault
257 .Lc2u_3rem8lp: tst ip, #8
258 movne r3, r7, lspull #24
259 ldmneia r1!, {r4, r7}
260 orrne r3, r3, r4, lspush #8
261 movne r4, r4, lspull #24
262 orrne r4, r4, r7, lspush #8
263 stmneia r0!, {r3 - r4} @ Shouldnt fault
265 movne r3, r7, lspull #24
267 orrne r3, r3, r7, lspush #8
268 TUSER( strne) r3, [r0], #4 @ Shouldnt fault
271 .Lc2u_3nowords: mov r3, r7, get_byte_3
275 USER( TUSER( strb) r3, [r0], #1) @ May fault
277 USER( TUSER( strgeb) r3, [r0], #1) @ May fault
279 USER( TUSER( strgtb) r3, [r0], #1) @ May fault
281 ENDPROC(__copy_to_user)
283 .pushsection .fixup,"ax"
285 9001: ldmfd sp!, {r0, r4 - r7, pc}
288 /* Prototype: unsigned long __copy_from_user(void *to,const void *from,unsigned long n);
289 * Purpose : copy a block from user memory to kernel memory
290 * Params : to - kernel memory
291 * : from - user memory
292 * : n - number of bytes to copy
293 * Returns : Number of bytes NOT copied.
295 .Lcfu_dest_not_aligned:
298 USER( TUSER( ldrb) r3, [r1], #1) @ May fault
300 USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault
302 USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault
307 ENTRY(__copy_from_user)
308 stmfd sp!, {r0, r2, r4 - r7, lr}
312 bne .Lcfu_dest_not_aligned
315 bne .Lcfu_src_not_aligned
318 * Seeing as there has to be at least 8 bytes to copy, we can
319 * copy one word, and force a user-mode page fault...
322 .Lcfu_0fupi: subs r2, r2, #4
325 USER( TUSER( ldr) r3, [r1], #4)
327 mov ip, r1, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
329 movs ip, ip, lsr #32 - PAGE_SHIFT
332 * ip = max no. of bytes to copy before needing another "strt" insn
340 .Lcfu_0cpy8lp: ldmia r1!, {r3 - r6} @ Shouldnt fault
342 ldmia r1!, {r3 - r6} @ Shouldnt fault
347 .Lcfu_0rem8lp: cmn ip, #16
348 ldmgeia r1!, {r3 - r6} @ Shouldnt fault
349 stmgeia r0!, {r3 - r6}
351 ldmneia r1!, {r3 - r4} @ Shouldnt fault
352 stmneia r0!, {r3 - r4}
354 TUSER( ldrne) r3, [r1], #4 @ Shouldnt fault
358 .Lcfu_0nowords: teq ip, #0
360 .Lcfu_nowords: cmp ip, #2
361 USER( TUSER( ldrb) r3, [r1], #1) @ May fault
363 USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault
365 USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault
372 .Lcfu_finished: mov r0, #0
374 ldmfd sp!, {r4 - r7, pc}
376 .Lcfu_src_not_aligned:
378 USER( TUSER( ldr) r7, [r1], #4) @ May fault
382 .Lcfu_1fupi: subs r2, r2, #4
385 mov r3, r7, lspull #8
386 USER( TUSER( ldr) r7, [r1], #4) @ May fault
387 orr r3, r3, r7, lspush #24
389 mov ip, r1, lsl #32 - PAGE_SHIFT
391 movs ip, ip, lsr #32 - PAGE_SHIFT
399 .Lcfu_1cpy8lp: mov r3, r7, lspull #8
400 ldmia r1!, {r4 - r7} @ Shouldnt fault
402 orr r3, r3, r4, lspush #24
403 mov r4, r4, lspull #8
404 orr r4, r4, r5, lspush #24
405 mov r5, r5, lspull #8
406 orr r5, r5, r6, lspush #24
407 mov r6, r6, lspull #8
408 orr r6, r6, r7, lspush #24
412 .Lcfu_1rem8lp: tst ip, #8
413 movne r3, r7, lspull #8
414 ldmneia r1!, {r4, r7} @ Shouldnt fault
415 orrne r3, r3, r4, lspush #24
416 movne r4, r4, lspull #8
417 orrne r4, r4, r7, lspush #24
418 stmneia r0!, {r3 - r4}
420 movne r3, r7, lspull #8
421 USER( TUSER( ldrne) r7, [r1], #4) @ May fault
422 orrne r3, r3, r7, lspush #24
426 .Lcfu_1nowords: mov r3, r7, get_byte_1
431 movge r3, r7, get_byte_2
433 movgt r3, r7, get_byte_3
437 .Lcfu_2fupi: subs r2, r2, #4
440 mov r3, r7, lspull #16
441 USER( TUSER( ldr) r7, [r1], #4) @ May fault
442 orr r3, r3, r7, lspush #16
444 mov ip, r1, lsl #32 - PAGE_SHIFT
446 movs ip, ip, lsr #32 - PAGE_SHIFT
455 .Lcfu_2cpy8lp: mov r3, r7, lspull #16
456 ldmia r1!, {r4 - r7} @ Shouldnt fault
458 orr r3, r3, r4, lspush #16
459 mov r4, r4, lspull #16
460 orr r4, r4, r5, lspush #16
461 mov r5, r5, lspull #16
462 orr r5, r5, r6, lspush #16
463 mov r6, r6, lspull #16
464 orr r6, r6, r7, lspush #16
468 .Lcfu_2rem8lp: tst ip, #8
469 movne r3, r7, lspull #16
470 ldmneia r1!, {r4, r7} @ Shouldnt fault
471 orrne r3, r3, r4, lspush #16
472 movne r4, r4, lspull #16
473 orrne r4, r4, r7, lspush #16
474 stmneia r0!, {r3 - r4}
476 movne r3, r7, lspull #16
477 USER( TUSER( ldrne) r7, [r1], #4) @ May fault
478 orrne r3, r3, r7, lspush #16
482 .Lcfu_2nowords: mov r3, r7, get_byte_2
487 movge r3, r7, get_byte_3
489 USER( TUSER( ldrgtb) r3, [r1], #0) @ May fault
493 .Lcfu_3fupi: subs r2, r2, #4
496 mov r3, r7, lspull #24
497 USER( TUSER( ldr) r7, [r1], #4) @ May fault
498 orr r3, r3, r7, lspush #8
500 mov ip, r1, lsl #32 - PAGE_SHIFT
502 movs ip, ip, lsr #32 - PAGE_SHIFT
510 .Lcfu_3cpy8lp: mov r3, r7, lspull #24
511 ldmia r1!, {r4 - r7} @ Shouldnt fault
512 orr r3, r3, r4, lspush #8
513 mov r4, r4, lspull #24
514 orr r4, r4, r5, lspush #8
515 mov r5, r5, lspull #24
516 orr r5, r5, r6, lspush #8
517 mov r6, r6, lspull #24
518 orr r6, r6, r7, lspush #8
523 .Lcfu_3rem8lp: tst ip, #8
524 movne r3, r7, lspull #24
525 ldmneia r1!, {r4, r7} @ Shouldnt fault
526 orrne r3, r3, r4, lspush #8
527 movne r4, r4, lspull #24
528 orrne r4, r4, r7, lspush #8
529 stmneia r0!, {r3 - r4}
531 movne r3, r7, lspull #24
532 USER( TUSER( ldrne) r7, [r1], #4) @ May fault
533 orrne r3, r3, r7, lspush #8
537 .Lcfu_3nowords: mov r3, r7, get_byte_3
542 USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault
544 USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault
547 ENDPROC(__copy_from_user)
549 .pushsection .fixup,"ax"
552 * We took an exception. r0 contains a pointer to
553 * the byte not copied.
555 9001: ldr r2, [sp], #4 @ void *to
556 sub r2, r0, r2 @ bytes copied
557 ldr r1, [sp], #4 @ unsigned long count
558 subs r4, r1, r2 @ bytes left to copy
562 ldmfd sp!, {r4 - r7, pc}