| blob | 5c908b1cb8ed5db3eeabfb89f7f659f99d2d6f76 |
1 /*
2 * linux/arch/arm/lib/uaccess.S
3 *
4 * Copyright (C) 1995, 1996,1997,1998 Russell King
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * Routines to block copy data to/from user memory
11 * These are highly optimised both for the 4k page size
12 * and for various alignments.
13 */
14 #include <linux/linkage.h>
15 #include <asm/assembler.h>
16 #include <asm/errno.h>
17 #include <asm/domain.h>
19 .text
21 #define PAGE_SHIFT 12
23 /* Prototype: int __copy_to_user(void *to, const char *from, size_t n)
24 * Purpose : copy a block to user memory from kernel memory
25 * Params : to - user memory
26 * : from - kernel memory
27 * : n - number of bytes to copy
28 * Returns : Number of bytes NOT copied.
29 */
31 .Lc2u_dest_not_aligned:
32 rsb ip, ip, #4
33 cmp ip, #2
34 ldrb r3, [r1], #1
35 USER( TUSER( strb) r3, [r0], #1) @ May fault
36 ldrgeb r3, [r1], #1
37 USER( TUSER( strgeb) r3, [r0], #1) @ May fault
38 ldrgtb r3, [r1], #1
39 USER( TUSER( strgtb) r3, [r0], #1) @ May fault
40 sub r2, r2, ip
41 b .Lc2u_dest_aligned
43 ENTRY(__copy_to_user)
44 stmfd sp!, {r2, r4 - r7, lr}
45 cmp r2, #4
46 blt .Lc2u_not_enough
47 ands ip, r0, #3
48 bne .Lc2u_dest_not_aligned
49 .Lc2u_dest_aligned:
51 ands ip, r1, #3
52 bne .Lc2u_src_not_aligned
53 /*
54 * Seeing as there has to be at least 8 bytes to copy, we can
55 * copy one word, and force a user-mode page fault...
56 */
58 .Lc2u_0fupi: subs r2, r2, #4
59 addmi ip, r2, #4
60 bmi .Lc2u_0nowords
61 ldr r3, [r1], #4
62 USER( TUSER( str) r3, [r0], #4) @ May fault
63 mov ip, r0, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
64 rsb ip, ip, #0
65 movs ip, ip, lsr #32 - PAGE_SHIFT
66 beq .Lc2u_0fupi
67 /*
68 * ip = max no. of bytes to copy before needing another "strt" insn
69 */
70 cmp r2, ip
71 movlt ip, r2
72 sub r2, r2, ip
73 subs ip, ip, #32
74 blt .Lc2u_0rem8lp
76 .Lc2u_0cpy8lp: ldmia r1!, {r3 - r6}
77 stmia r0!, {r3 - r6} @ Shouldnt fault
78 ldmia r1!, {r3 - r6}
79 subs ip, ip, #32
80 stmia r0!, {r3 - r6} @ Shouldnt fault
81 bpl .Lc2u_0cpy8lp
83 .Lc2u_0rem8lp: cmn ip, #16
84 ldmgeia r1!, {r3 - r6}
85 stmgeia r0!, {r3 - r6} @ Shouldnt fault
86 tst ip, #8
87 ldmneia r1!, {r3 - r4}
88 stmneia r0!, {r3 - r4} @ Shouldnt fault
89 tst ip, #4
90 ldrne r3, [r1], #4
91 TUSER( strne) r3, [r0], #4 @ Shouldnt fault
92 ands ip, ip, #3
93 beq .Lc2u_0fupi
94 .Lc2u_0nowords: teq ip, #0
95 beq .Lc2u_finished
96 .Lc2u_nowords: cmp ip, #2
97 ldrb r3, [r1], #1
98 USER( TUSER( strb) r3, [r0], #1) @ May fault
99 ldrgeb r3, [r1], #1
100 USER( TUSER( strgeb) r3, [r0], #1) @ May fault
101 ldrgtb r3, [r1], #1
102 USER( TUSER( strgtb) r3, [r0], #1) @ May fault
103 b .Lc2u_finished
105 .Lc2u_not_enough:
106 movs ip, r2
107 bne .Lc2u_nowords
108 .Lc2u_finished: mov r0, #0
109 ldmfd sp!, {r2, r4 - r7, pc}
111 .Lc2u_src_not_aligned:
112 bic r1, r1, #3
113 ldr r7, [r1], #4
114 cmp ip, #2
115 bgt .Lc2u_3fupi
116 beq .Lc2u_2fupi
117 .Lc2u_1fupi: subs r2, r2, #4
118 addmi ip, r2, #4
119 bmi .Lc2u_1nowords
120 mov r3, r7, pull #8
121 ldr r7, [r1], #4
122 orr r3, r3, r7, push #24
123 USER( TUSER( str) r3, [r0], #4) @ May fault
124 mov ip, r0, lsl #32 - PAGE_SHIFT
125 rsb ip, ip, #0
126 movs ip, ip, lsr #32 - PAGE_SHIFT
127 beq .Lc2u_1fupi
128 cmp r2, ip
129 movlt ip, r2
130 sub r2, r2, ip
131 subs ip, ip, #16
132 blt .Lc2u_1rem8lp
134 .Lc2u_1cpy8lp: mov r3, r7, pull #8
135 ldmia r1!, {r4 - r7}
136 subs ip, ip, #16
137 orr r3, r3, r4, push #24
138 mov r4, r4, pull #8
139 orr r4, r4, r5, push #24
140 mov r5, r5, pull #8
141 orr r5, r5, r6, push #24
142 mov r6, r6, pull #8
143 orr r6, r6, r7, push #24
144 stmia r0!, {r3 - r6} @ Shouldnt fault
145 bpl .Lc2u_1cpy8lp
147 .Lc2u_1rem8lp: tst ip, #8
148 movne r3, r7, pull #8
149 ldmneia r1!, {r4, r7}
150 orrne r3, r3, r4, push #24
151 movne r4, r4, pull #8
152 orrne r4, r4, r7, push #24
153 stmneia r0!, {r3 - r4} @ Shouldnt fault
154 tst ip, #4
155 movne r3, r7, pull #8
156 ldrne r7, [r1], #4
157 orrne r3, r3, r7, push #24
158 TUSER( strne) r3, [r0], #4 @ Shouldnt fault
159 ands ip, ip, #3
160 beq .Lc2u_1fupi
161 .Lc2u_1nowords: mov r3, r7, get_byte_1
162 teq ip, #0
163 beq .Lc2u_finished
164 cmp ip, #2
165 USER( TUSER( strb) r3, [r0], #1) @ May fault
166 movge r3, r7, get_byte_2
167 USER( TUSER( strgeb) r3, [r0], #1) @ May fault
168 movgt r3, r7, get_byte_3
169 USER( TUSER( strgtb) r3, [r0], #1) @ May fault
170 b .Lc2u_finished
172 .Lc2u_2fupi: subs r2, r2, #4
173 addmi ip, r2, #4
174 bmi .Lc2u_2nowords
175 mov r3, r7, pull #16
176 ldr r7, [r1], #4
177 orr r3, r3, r7, push #16
178 USER( TUSER( str) r3, [r0], #4) @ May fault
179 mov ip, r0, lsl #32 - PAGE_SHIFT
180 rsb ip, ip, #0
181 movs ip, ip, lsr #32 - PAGE_SHIFT
182 beq .Lc2u_2fupi
183 cmp r2, ip
184 movlt ip, r2
185 sub r2, r2, ip
186 subs ip, ip, #16
187 blt .Lc2u_2rem8lp
189 .Lc2u_2cpy8lp: mov r3, r7, pull #16
190 ldmia r1!, {r4 - r7}
191 subs ip, ip, #16
192 orr r3, r3, r4, push #16
193 mov r4, r4, pull #16
194 orr r4, r4, r5, push #16
195 mov r5, r5, pull #16
196 orr r5, r5, r6, push #16
197 mov r6, r6, pull #16
198 orr r6, r6, r7, push #16
199 stmia r0!, {r3 - r6} @ Shouldnt fault
200 bpl .Lc2u_2cpy8lp
202 .Lc2u_2rem8lp: tst ip, #8
203 movne r3, r7, pull #16
204 ldmneia r1!, {r4, r7}
205 orrne r3, r3, r4, push #16
206 movne r4, r4, pull #16
207 orrne r4, r4, r7, push #16
208 stmneia r0!, {r3 - r4} @ Shouldnt fault
209 tst ip, #4
210 movne r3, r7, pull #16
211 ldrne r7, [r1], #4
212 orrne r3, r3, r7, push #16
213 TUSER( strne) r3, [r0], #4 @ Shouldnt fault
214 ands ip, ip, #3
215 beq .Lc2u_2fupi
216 .Lc2u_2nowords: mov r3, r7, get_byte_2
217 teq ip, #0
218 beq .Lc2u_finished
219 cmp ip, #2
220 USER( TUSER( strb) r3, [r0], #1) @ May fault
221 movge r3, r7, get_byte_3
222 USER( TUSER( strgeb) r3, [r0], #1) @ May fault
223 ldrgtb r3, [r1], #0
224 USER( TUSER( strgtb) r3, [r0], #1) @ May fault
225 b .Lc2u_finished
227 .Lc2u_3fupi: subs r2, r2, #4
228 addmi ip, r2, #4
229 bmi .Lc2u_3nowords
230 mov r3, r7, pull #24
231 ldr r7, [r1], #4
232 orr r3, r3, r7, push #8
233 USER( TUSER( str) r3, [r0], #4) @ May fault
234 mov ip, r0, lsl #32 - PAGE_SHIFT
235 rsb ip, ip, #0
236 movs ip, ip, lsr #32 - PAGE_SHIFT
237 beq .Lc2u_3fupi
238 cmp r2, ip
239 movlt ip, r2
240 sub r2, r2, ip
241 subs ip, ip, #16
242 blt .Lc2u_3rem8lp
244 .Lc2u_3cpy8lp: mov r3, r7, pull #24
245 ldmia r1!, {r4 - r7}
246 subs ip, ip, #16
247 orr r3, r3, r4, push #8
248 mov r4, r4, pull #24
249 orr r4, r4, r5, push #8
250 mov r5, r5, pull #24
251 orr r5, r5, r6, push #8
252 mov r6, r6, pull #24
253 orr r6, r6, r7, push #8
254 stmia r0!, {r3 - r6} @ Shouldnt fault
255 bpl .Lc2u_3cpy8lp
257 .Lc2u_3rem8lp: tst ip, #8
258 movne r3, r7, pull #24
259 ldmneia r1!, {r4, r7}
260 orrne r3, r3, r4, push #8
261 movne r4, r4, pull #24
262 orrne r4, r4, r7, push #8
263 stmneia r0!, {r3 - r4} @ Shouldnt fault
264 tst ip, #4
265 movne r3, r7, pull #24
266 ldrne r7, [r1], #4
267 orrne r3, r3, r7, push #8
268 TUSER( strne) r3, [r0], #4 @ Shouldnt fault
269 ands ip, ip, #3
270 beq .Lc2u_3fupi
271 .Lc2u_3nowords: mov r3, r7, get_byte_3
272 teq ip, #0
273 beq .Lc2u_finished
274 cmp ip, #2
275 USER( TUSER( strb) r3, [r0], #1) @ May fault
276 ldrgeb r3, [r1], #1
277 USER( TUSER( strgeb) r3, [r0], #1) @ May fault
278 ldrgtb r3, [r1], #0
279 USER( TUSER( strgtb) r3, [r0], #1) @ May fault
280 b .Lc2u_finished
281 ENDPROC(__copy_to_user)
283 .pushsection .fixup,"ax"
284 .align 0
285 9001: ldmfd sp!, {r0, r4 - r7, pc}
286 .popsection
288 /* Prototype: unsigned long __copy_from_user(void *to,const void *from,unsigned long n);
289 * Purpose : copy a block from user memory to kernel memory
290 * Params : to - kernel memory
291 * : from - user memory
292 * : n - number of bytes to copy
293 * Returns : Number of bytes NOT copied.
294 */
295 .Lcfu_dest_not_aligned:
296 rsb ip, ip, #4
297 cmp ip, #2
298 USER( TUSER( ldrb) r3, [r1], #1) @ May fault
299 strb r3, [r0], #1
300 USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault
301 strgeb r3, [r0], #1
302 USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault
303 strgtb r3, [r0], #1
304 sub r2, r2, ip
305 b .Lcfu_dest_aligned
307 ENTRY(__copy_from_user)
308 stmfd sp!, {r0, r2, r4 - r7, lr}
309 cmp r2, #4
310 blt .Lcfu_not_enough
311 ands ip, r0, #3
312 bne .Lcfu_dest_not_aligned
313 .Lcfu_dest_aligned:
314 ands ip, r1, #3
315 bne .Lcfu_src_not_aligned
317 /*
318 * Seeing as there has to be at least 8 bytes to copy, we can
319 * copy one word, and force a user-mode page fault...
320 */
322 .Lcfu_0fupi: subs r2, r2, #4
323 addmi ip, r2, #4
324 bmi .Lcfu_0nowords
325 USER( TUSER( ldr) r3, [r1], #4)
326 str r3, [r0], #4
327 mov ip, r1, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
328 rsb ip, ip, #0
329 movs ip, ip, lsr #32 - PAGE_SHIFT
330 beq .Lcfu_0fupi
331 /*
332 * ip = max no. of bytes to copy before needing another "strt" insn
333 */
334 cmp r2, ip
335 movlt ip, r2
336 sub r2, r2, ip
337 subs ip, ip, #32
338 blt .Lcfu_0rem8lp
340 .Lcfu_0cpy8lp: ldmia r1!, {r3 - r6} @ Shouldnt fault
341 stmia r0!, {r3 - r6}
342 ldmia r1!, {r3 - r6} @ Shouldnt fault
343 subs ip, ip, #32
344 stmia r0!, {r3 - r6}
345 bpl .Lcfu_0cpy8lp
347 .Lcfu_0rem8lp: cmn ip, #16
348 ldmgeia r1!, {r3 - r6} @ Shouldnt fault
349 stmgeia r0!, {r3 - r6}
350 tst ip, #8
351 ldmneia r1!, {r3 - r4} @ Shouldnt fault
352 stmneia r0!, {r3 - r4}
353 tst ip, #4
354 TUSER( ldrne) r3, [r1], #4 @ Shouldnt fault
355 strne r3, [r0], #4
356 ands ip, ip, #3
357 beq .Lcfu_0fupi
358 .Lcfu_0nowords: teq ip, #0
359 beq .Lcfu_finished
360 .Lcfu_nowords: cmp ip, #2
361 USER( TUSER( ldrb) r3, [r1], #1) @ May fault
362 strb r3, [r0], #1
363 USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault
364 strgeb r3, [r0], #1
365 USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault
366 strgtb r3, [r0], #1
367 b .Lcfu_finished
369 .Lcfu_not_enough:
370 movs ip, r2
371 bne .Lcfu_nowords
372 .Lcfu_finished: mov r0, #0
373 add sp, sp, #8
374 ldmfd sp!, {r4 - r7, pc}
376 .Lcfu_src_not_aligned:
377 bic r1, r1, #3
378 USER( TUSER( ldr) r7, [r1], #4) @ May fault
379 cmp ip, #2
380 bgt .Lcfu_3fupi
381 beq .Lcfu_2fupi
382 .Lcfu_1fupi: subs r2, r2, #4
383 addmi ip, r2, #4
384 bmi .Lcfu_1nowords
385 mov r3, r7, pull #8
386 USER( TUSER( ldr) r7, [r1], #4) @ May fault
387 orr r3, r3, r7, push #24
388 str r3, [r0], #4
389 mov ip, r1, lsl #32 - PAGE_SHIFT
390 rsb ip, ip, #0
391 movs ip, ip, lsr #32 - PAGE_SHIFT
392 beq .Lcfu_1fupi
393 cmp r2, ip
394 movlt ip, r2
395 sub r2, r2, ip
396 subs ip, ip, #16
397 blt .Lcfu_1rem8lp
399 .Lcfu_1cpy8lp: mov r3, r7, pull #8
400 ldmia r1!, {r4 - r7} @ Shouldnt fault
401 subs ip, ip, #16
402 orr r3, r3, r4, push #24
403 mov r4, r4, pull #8
404 orr r4, r4, r5, push #24
405 mov r5, r5, pull #8
406 orr r5, r5, r6, push #24
407 mov r6, r6, pull #8
408 orr r6, r6, r7, push #24
409 stmia r0!, {r3 - r6}
410 bpl .Lcfu_1cpy8lp
412 .Lcfu_1rem8lp: tst ip, #8
413 movne r3, r7, pull #8
414 ldmneia r1!, {r4, r7} @ Shouldnt fault
415 orrne r3, r3, r4, push #24
416 movne r4, r4, pull #8
417 orrne r4, r4, r7, push #24
418 stmneia r0!, {r3 - r4}
419 tst ip, #4
420 movne r3, r7, pull #8
421 USER( TUSER( ldrne) r7, [r1], #4) @ May fault
422 orrne r3, r3, r7, push #24
423 strne r3, [r0], #4
424 ands ip, ip, #3
425 beq .Lcfu_1fupi
426 .Lcfu_1nowords: mov r3, r7, get_byte_1
427 teq ip, #0
428 beq .Lcfu_finished
429 cmp ip, #2
430 strb r3, [r0], #1
431 movge r3, r7, get_byte_2
432 strgeb r3, [r0], #1
433 movgt r3, r7, get_byte_3
434 strgtb r3, [r0], #1
435 b .Lcfu_finished
437 .Lcfu_2fupi: subs r2, r2, #4
438 addmi ip, r2, #4
439 bmi .Lcfu_2nowords
440 mov r3, r7, pull #16
441 USER( TUSER( ldr) r7, [r1], #4) @ May fault
442 orr r3, r3, r7, push #16
443 str r3, [r0], #4
444 mov ip, r1, lsl #32 - PAGE_SHIFT
445 rsb ip, ip, #0
446 movs ip, ip, lsr #32 - PAGE_SHIFT
447 beq .Lcfu_2fupi
448 cmp r2, ip
449 movlt ip, r2
450 sub r2, r2, ip
451 subs ip, ip, #16
452 blt .Lcfu_2rem8lp
455 .Lcfu_2cpy8lp: mov r3, r7, pull #16
456 ldmia r1!, {r4 - r7} @ Shouldnt fault
457 subs ip, ip, #16
458 orr r3, r3, r4, push #16
459 mov r4, r4, pull #16
460 orr r4, r4, r5, push #16
461 mov r5, r5, pull #16
462 orr r5, r5, r6, push #16
463 mov r6, r6, pull #16
464 orr r6, r6, r7, push #16
465 stmia r0!, {r3 - r6}
466 bpl .Lcfu_2cpy8lp
468 .Lcfu_2rem8lp: tst ip, #8
469 movne r3, r7, pull #16
470 ldmneia r1!, {r4, r7} @ Shouldnt fault
471 orrne r3, r3, r4, push #16
472 movne r4, r4, pull #16
473 orrne r4, r4, r7, push #16
474 stmneia r0!, {r3 - r4}
475 tst ip, #4
476 movne r3, r7, pull #16
477 USER( TUSER( ldrne) r7, [r1], #4) @ May fault
478 orrne r3, r3, r7, push #16
479 strne r3, [r0], #4
480 ands ip, ip, #3
481 beq .Lcfu_2fupi
482 .Lcfu_2nowords: mov r3, r7, get_byte_2
483 teq ip, #0
484 beq .Lcfu_finished
485 cmp ip, #2
486 strb r3, [r0], #1
487 movge r3, r7, get_byte_3
488 strgeb r3, [r0], #1
489 USER( TUSER( ldrgtb) r3, [r1], #0) @ May fault
490 strgtb r3, [r0], #1
491 b .Lcfu_finished
493 .Lcfu_3fupi: subs r2, r2, #4
494 addmi ip, r2, #4
495 bmi .Lcfu_3nowords
496 mov r3, r7, pull #24
497 USER( TUSER( ldr) r7, [r1], #4) @ May fault
498 orr r3, r3, r7, push #8
499 str r3, [r0], #4
500 mov ip, r1, lsl #32 - PAGE_SHIFT
501 rsb ip, ip, #0
502 movs ip, ip, lsr #32 - PAGE_SHIFT
503 beq .Lcfu_3fupi
504 cmp r2, ip
505 movlt ip, r2
506 sub r2, r2, ip
507 subs ip, ip, #16
508 blt .Lcfu_3rem8lp
510 .Lcfu_3cpy8lp: mov r3, r7, pull #24
511 ldmia r1!, {r4 - r7} @ Shouldnt fault
512 orr r3, r3, r4, push #8
513 mov r4, r4, pull #24
514 orr r4, r4, r5, push #8
515 mov r5, r5, pull #24
516 orr r5, r5, r6, push #8
517 mov r6, r6, pull #24
518 orr r6, r6, r7, push #8
519 stmia r0!, {r3 - r6}
520 subs ip, ip, #16
521 bpl .Lcfu_3cpy8lp
523 .Lcfu_3rem8lp: tst ip, #8
524 movne r3, r7, pull #24
525 ldmneia r1!, {r4, r7} @ Shouldnt fault
526 orrne r3, r3, r4, push #8
527 movne r4, r4, pull #24
528 orrne r4, r4, r7, push #8
529 stmneia r0!, {r3 - r4}
530 tst ip, #4
531 movne r3, r7, pull #24
532 USER( TUSER( ldrne) r7, [r1], #4) @ May fault
533 orrne r3, r3, r7, push #8
534 strne r3, [r0], #4
535 ands ip, ip, #3
536 beq .Lcfu_3fupi
537 .Lcfu_3nowords: mov r3, r7, get_byte_3
538 teq ip, #0
539 beq .Lcfu_finished
540 cmp ip, #2
541 strb r3, [r0], #1
542 USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault
543 strgeb r3, [r0], #1
544 USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault
545 strgtb r3, [r0], #1
546 b .Lcfu_finished
547 ENDPROC(__copy_from_user)
549 .pushsection .fixup,"ax"
550 .align 0
551 /*
552 * We took an exception. r0 contains a pointer to
553 * the byte not copied.
554 */
555 9001: ldr r2, [sp], #4 @ void *to
556 sub r2, r0, r2 @ bytes copied
557 ldr r1, [sp], #4 @ unsigned long count
558 subs r4, r1, r2 @ bytes left to copy
559 movne r1, r4
560 blne __memzero
561 mov r0, r4
562 ldmfd sp!, {r4 - r7, pc}
563 .popsection