| blob | ffdd27498ceef69471f230317725ecea1951f111 |
1 /*
2 * linux/arch/arm/lib/uaccess.S
3 *
4 * Copyright (C) 1995, 1996,1997,1998 Russell King
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * Routines to block copy data to/from user memory
11 * These are highly optimised both for the 4k page size
12 * and for various alignments.
13 */
14 #include <linux/linkage.h>
15 #include <asm/assembler.h>
16 #include <asm/errno.h>
18 .text
20 #define PAGE_SHIFT 12
22 /* Prototype: int __copy_to_user(void *to, const char *from, size_t n)
23 * Purpose : copy a block to user memory from kernel memory
24 * Params : to - user memory
25 * : from - kernel memory
26 * : n - number of bytes to copy
27 * Returns : Number of bytes NOT copied.
28 */
30 .Lc2u_dest_not_aligned:
31 rsb ip, ip, #4
32 cmp ip, #2
33 ldrb r3, [r1], #1
34 USER( strbt r3, [r0], #1) @ May fault
35 ldrgeb r3, [r1], #1
36 USER( strgebt r3, [r0], #1) @ May fault
37 ldrgtb r3, [r1], #1
38 USER( strgtbt r3, [r0], #1) @ May fault
39 sub r2, r2, ip
40 b .Lc2u_dest_aligned
42 ENTRY(__copy_to_user)
43 stmfd sp!, {r2, r4 - r7, lr}
44 cmp r2, #4
45 blt .Lc2u_not_enough
46 ands ip, r0, #3
47 bne .Lc2u_dest_not_aligned
48 .Lc2u_dest_aligned:
50 ands ip, r1, #3
51 bne .Lc2u_src_not_aligned
52 /*
53 * Seeing as there has to be at least 8 bytes to copy, we can
54 * copy one word, and force a user-mode page fault...
55 */
57 .Lc2u_0fupi: subs r2, r2, #4
58 addmi ip, r2, #4
59 bmi .Lc2u_0nowords
60 ldr r3, [r1], #4
61 USER( strt r3, [r0], #4) @ May fault
62 mov ip, r0, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
63 rsb ip, ip, #0
64 movs ip, ip, lsr #32 - PAGE_SHIFT
65 beq .Lc2u_0fupi
66 /*
67 * ip = max no. of bytes to copy before needing another "strt" insn
68 */
69 cmp r2, ip
70 movlt ip, r2
71 sub r2, r2, ip
72 subs ip, ip, #32
73 blt .Lc2u_0rem8lp
75 .Lc2u_0cpy8lp: ldmia r1!, {r3 - r6}
76 stmia r0!, {r3 - r6} @ Shouldnt fault
77 ldmia r1!, {r3 - r6}
78 subs ip, ip, #32
79 stmia r0!, {r3 - r6} @ Shouldnt fault
80 bpl .Lc2u_0cpy8lp
82 .Lc2u_0rem8lp: cmn ip, #16
83 ldmgeia r1!, {r3 - r6}
84 stmgeia r0!, {r3 - r6} @ Shouldnt fault
85 tst ip, #8
86 ldmneia r1!, {r3 - r4}
87 stmneia r0!, {r3 - r4} @ Shouldnt fault
88 tst ip, #4
89 ldrne r3, [r1], #4
90 strnet r3, [r0], #4 @ Shouldnt fault
91 ands ip, ip, #3
92 beq .Lc2u_0fupi
93 .Lc2u_0nowords: teq ip, #0
94 beq .Lc2u_finished
95 .Lc2u_nowords: cmp ip, #2
96 ldrb r3, [r1], #1
97 USER( strbt r3, [r0], #1) @ May fault
98 ldrgeb r3, [r1], #1
99 USER( strgebt r3, [r0], #1) @ May fault
100 ldrgtb r3, [r1], #1
101 USER( strgtbt r3, [r0], #1) @ May fault
102 b .Lc2u_finished
104 .Lc2u_not_enough:
105 movs ip, r2
106 bne .Lc2u_nowords
107 .Lc2u_finished: mov r0, #0
108 ldmfd sp!, {r2, r4 - r7, pc}
110 .Lc2u_src_not_aligned:
111 bic r1, r1, #3
112 ldr r7, [r1], #4
113 cmp ip, #2
114 bgt .Lc2u_3fupi
115 beq .Lc2u_2fupi
116 .Lc2u_1fupi: subs r2, r2, #4
117 addmi ip, r2, #4
118 bmi .Lc2u_1nowords
119 mov r3, r7, pull #8
120 ldr r7, [r1], #4
121 orr r3, r3, r7, push #24
122 USER( strt r3, [r0], #4) @ May fault
123 mov ip, r0, lsl #32 - PAGE_SHIFT
124 rsb ip, ip, #0
125 movs ip, ip, lsr #32 - PAGE_SHIFT
126 beq .Lc2u_1fupi
127 cmp r2, ip
128 movlt ip, r2
129 sub r2, r2, ip
130 subs ip, ip, #16
131 blt .Lc2u_1rem8lp
133 .Lc2u_1cpy8lp: mov r3, r7, pull #8
134 ldmia r1!, {r4 - r7}
135 subs ip, ip, #16
136 orr r3, r3, r4, push #24
137 mov r4, r4, pull #8
138 orr r4, r4, r5, push #24
139 mov r5, r5, pull #8
140 orr r5, r5, r6, push #24
141 mov r6, r6, pull #8
142 orr r6, r6, r7, push #24
143 stmia r0!, {r3 - r6} @ Shouldnt fault
144 bpl .Lc2u_1cpy8lp
146 .Lc2u_1rem8lp: tst ip, #8
147 movne r3, r7, pull #8
148 ldmneia r1!, {r4, r7}
149 orrne r3, r3, r4, push #24
150 movne r4, r4, pull #8
151 orrne r4, r4, r7, push #24
152 stmneia r0!, {r3 - r4} @ Shouldnt fault
153 tst ip, #4
154 movne r3, r7, pull #8
155 ldrne r7, [r1], #4
156 orrne r3, r3, r7, push #24
157 strnet r3, [r0], #4 @ Shouldnt fault
158 ands ip, ip, #3
159 beq .Lc2u_1fupi
160 .Lc2u_1nowords: mov r3, r7, get_byte_1
161 teq ip, #0
162 beq .Lc2u_finished
163 cmp ip, #2
164 USER( strbt r3, [r0], #1) @ May fault
165 movge r3, r7, get_byte_2
166 USER( strgebt r3, [r0], #1) @ May fault
167 movgt r3, r7, get_byte_3
168 USER( strgtbt r3, [r0], #1) @ May fault
169 b .Lc2u_finished
171 .Lc2u_2fupi: subs r2, r2, #4
172 addmi ip, r2, #4
173 bmi .Lc2u_2nowords
174 mov r3, r7, pull #16
175 ldr r7, [r1], #4
176 orr r3, r3, r7, push #16
177 USER( strt r3, [r0], #4) @ May fault
178 mov ip, r0, lsl #32 - PAGE_SHIFT
179 rsb ip, ip, #0
180 movs ip, ip, lsr #32 - PAGE_SHIFT
181 beq .Lc2u_2fupi
182 cmp r2, ip
183 movlt ip, r2
184 sub r2, r2, ip
185 subs ip, ip, #16
186 blt .Lc2u_2rem8lp
188 .Lc2u_2cpy8lp: mov r3, r7, pull #16
189 ldmia r1!, {r4 - r7}
190 subs ip, ip, #16
191 orr r3, r3, r4, push #16
192 mov r4, r4, pull #16
193 orr r4, r4, r5, push #16
194 mov r5, r5, pull #16
195 orr r5, r5, r6, push #16
196 mov r6, r6, pull #16
197 orr r6, r6, r7, push #16
198 stmia r0!, {r3 - r6} @ Shouldnt fault
199 bpl .Lc2u_2cpy8lp
201 .Lc2u_2rem8lp: tst ip, #8
202 movne r3, r7, pull #16
203 ldmneia r1!, {r4, r7}
204 orrne r3, r3, r4, push #16
205 movne r4, r4, pull #16
206 orrne r4, r4, r7, push #16
207 stmneia r0!, {r3 - r4} @ Shouldnt fault
208 tst ip, #4
209 movne r3, r7, pull #16
210 ldrne r7, [r1], #4
211 orrne r3, r3, r7, push #16
212 strnet r3, [r0], #4 @ Shouldnt fault
213 ands ip, ip, #3
214 beq .Lc2u_2fupi
215 .Lc2u_2nowords: mov r3, r7, get_byte_2
216 teq ip, #0
217 beq .Lc2u_finished
218 cmp ip, #2
219 USER( strbt r3, [r0], #1) @ May fault
220 movge r3, r7, get_byte_3
221 USER( strgebt r3, [r0], #1) @ May fault
222 ldrgtb r3, [r1], #0
223 USER( strgtbt r3, [r0], #1) @ May fault
224 b .Lc2u_finished
226 .Lc2u_3fupi: subs r2, r2, #4
227 addmi ip, r2, #4
228 bmi .Lc2u_3nowords
229 mov r3, r7, pull #24
230 ldr r7, [r1], #4
231 orr r3, r3, r7, push #8
232 USER( strt r3, [r0], #4) @ May fault
233 mov ip, r0, lsl #32 - PAGE_SHIFT
234 rsb ip, ip, #0
235 movs ip, ip, lsr #32 - PAGE_SHIFT
236 beq .Lc2u_3fupi
237 cmp r2, ip
238 movlt ip, r2
239 sub r2, r2, ip
240 subs ip, ip, #16
241 blt .Lc2u_3rem8lp
243 .Lc2u_3cpy8lp: mov r3, r7, pull #24
244 ldmia r1!, {r4 - r7}
245 subs ip, ip, #16
246 orr r3, r3, r4, push #8
247 mov r4, r4, pull #24
248 orr r4, r4, r5, push #8
249 mov r5, r5, pull #24
250 orr r5, r5, r6, push #8
251 mov r6, r6, pull #24
252 orr r6, r6, r7, push #8
253 stmia r0!, {r3 - r6} @ Shouldnt fault
254 bpl .Lc2u_3cpy8lp
256 .Lc2u_3rem8lp: tst ip, #8
257 movne r3, r7, pull #24
258 ldmneia r1!, {r4, r7}
259 orrne r3, r3, r4, push #8
260 movne r4, r4, pull #24
261 orrne r4, r4, r7, push #8
262 stmneia r0!, {r3 - r4} @ Shouldnt fault
263 tst ip, #4
264 movne r3, r7, pull #24
265 ldrne r7, [r1], #4
266 orrne r3, r3, r7, push #8
267 strnet r3, [r0], #4 @ Shouldnt fault
268 ands ip, ip, #3
269 beq .Lc2u_3fupi
270 .Lc2u_3nowords: mov r3, r7, get_byte_3
271 teq ip, #0
272 beq .Lc2u_finished
273 cmp ip, #2
274 USER( strbt r3, [r0], #1) @ May fault
275 ldrgeb r3, [r1], #1
276 USER( strgebt r3, [r0], #1) @ May fault
277 ldrgtb r3, [r1], #0
278 USER( strgtbt r3, [r0], #1) @ May fault
279 b .Lc2u_finished
280 ENDPROC(__copy_to_user)
282 .section .fixup,"ax"
283 .align 0
284 9001: ldmfd sp!, {r0, r4 - r7, pc}
285 .previous
287 /* Prototype: unsigned long __copy_from_user(void *to,const void *from,unsigned long n);
288 * Purpose : copy a block from user memory to kernel memory
289 * Params : to - kernel memory
290 * : from - user memory
291 * : n - number of bytes to copy
292 * Returns : Number of bytes NOT copied.
293 */
294 .Lcfu_dest_not_aligned:
295 rsb ip, ip, #4
296 cmp ip, #2
297 USER( ldrbt r3, [r1], #1) @ May fault
298 strb r3, [r0], #1
299 USER( ldrgebt r3, [r1], #1) @ May fault
300 strgeb r3, [r0], #1
301 USER( ldrgtbt r3, [r1], #1) @ May fault
302 strgtb r3, [r0], #1
303 sub r2, r2, ip
304 b .Lcfu_dest_aligned
306 ENTRY(__copy_from_user)
307 stmfd sp!, {r0, r2, r4 - r7, lr}
308 cmp r2, #4
309 blt .Lcfu_not_enough
310 ands ip, r0, #3
311 bne .Lcfu_dest_not_aligned
312 .Lcfu_dest_aligned:
313 ands ip, r1, #3
314 bne .Lcfu_src_not_aligned
316 /*
317 * Seeing as there has to be at least 8 bytes to copy, we can
318 * copy one word, and force a user-mode page fault...
319 */
321 .Lcfu_0fupi: subs r2, r2, #4
322 addmi ip, r2, #4
323 bmi .Lcfu_0nowords
324 USER( ldrt r3, [r1], #4)
325 str r3, [r0], #4
326 mov ip, r1, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
327 rsb ip, ip, #0
328 movs ip, ip, lsr #32 - PAGE_SHIFT
329 beq .Lcfu_0fupi
330 /*
331 * ip = max no. of bytes to copy before needing another "strt" insn
332 */
333 cmp r2, ip
334 movlt ip, r2
335 sub r2, r2, ip
336 subs ip, ip, #32
337 blt .Lcfu_0rem8lp
339 .Lcfu_0cpy8lp: ldmia r1!, {r3 - r6} @ Shouldnt fault
340 stmia r0!, {r3 - r6}
341 ldmia r1!, {r3 - r6} @ Shouldnt fault
342 subs ip, ip, #32
343 stmia r0!, {r3 - r6}
344 bpl .Lcfu_0cpy8lp
346 .Lcfu_0rem8lp: cmn ip, #16
347 ldmgeia r1!, {r3 - r6} @ Shouldnt fault
348 stmgeia r0!, {r3 - r6}
349 tst ip, #8
350 ldmneia r1!, {r3 - r4} @ Shouldnt fault
351 stmneia r0!, {r3 - r4}
352 tst ip, #4
353 ldrnet r3, [r1], #4 @ Shouldnt fault
354 strne r3, [r0], #4
355 ands ip, ip, #3
356 beq .Lcfu_0fupi
357 .Lcfu_0nowords: teq ip, #0
358 beq .Lcfu_finished
359 .Lcfu_nowords: cmp ip, #2
360 USER( ldrbt r3, [r1], #1) @ May fault
361 strb r3, [r0], #1
362 USER( ldrgebt r3, [r1], #1) @ May fault
363 strgeb r3, [r0], #1
364 USER( ldrgtbt r3, [r1], #1) @ May fault
365 strgtb r3, [r0], #1
366 b .Lcfu_finished
368 .Lcfu_not_enough:
369 movs ip, r2
370 bne .Lcfu_nowords
371 .Lcfu_finished: mov r0, #0
372 add sp, sp, #8
373 ldmfd sp!, {r4 - r7, pc}
375 .Lcfu_src_not_aligned:
376 bic r1, r1, #3
377 USER( ldrt r7, [r1], #4) @ May fault
378 cmp ip, #2
379 bgt .Lcfu_3fupi
380 beq .Lcfu_2fupi
381 .Lcfu_1fupi: subs r2, r2, #4
382 addmi ip, r2, #4
383 bmi .Lcfu_1nowords
384 mov r3, r7, pull #8
385 USER( ldrt r7, [r1], #4) @ May fault
386 orr r3, r3, r7, push #24
387 str r3, [r0], #4
388 mov ip, r1, lsl #32 - PAGE_SHIFT
389 rsb ip, ip, #0
390 movs ip, ip, lsr #32 - PAGE_SHIFT
391 beq .Lcfu_1fupi
392 cmp r2, ip
393 movlt ip, r2
394 sub r2, r2, ip
395 subs ip, ip, #16
396 blt .Lcfu_1rem8lp
398 .Lcfu_1cpy8lp: mov r3, r7, pull #8
399 ldmia r1!, {r4 - r7} @ Shouldnt fault
400 subs ip, ip, #16
401 orr r3, r3, r4, push #24
402 mov r4, r4, pull #8
403 orr r4, r4, r5, push #24
404 mov r5, r5, pull #8
405 orr r5, r5, r6, push #24
406 mov r6, r6, pull #8
407 orr r6, r6, r7, push #24
408 stmia r0!, {r3 - r6}
409 bpl .Lcfu_1cpy8lp
411 .Lcfu_1rem8lp: tst ip, #8
412 movne r3, r7, pull #8
413 ldmneia r1!, {r4, r7} @ Shouldnt fault
414 orrne r3, r3, r4, push #24
415 movne r4, r4, pull #8
416 orrne r4, r4, r7, push #24
417 stmneia r0!, {r3 - r4}
418 tst ip, #4
419 movne r3, r7, pull #8
420 USER( ldrnet r7, [r1], #4) @ May fault
421 orrne r3, r3, r7, push #24
422 strne r3, [r0], #4
423 ands ip, ip, #3
424 beq .Lcfu_1fupi
425 .Lcfu_1nowords: mov r3, r7, get_byte_1
426 teq ip, #0
427 beq .Lcfu_finished
428 cmp ip, #2
429 strb r3, [r0], #1
430 movge r3, r7, get_byte_2
431 strgeb r3, [r0], #1
432 movgt r3, r7, get_byte_3
433 strgtb r3, [r0], #1
434 b .Lcfu_finished
436 .Lcfu_2fupi: subs r2, r2, #4
437 addmi ip, r2, #4
438 bmi .Lcfu_2nowords
439 mov r3, r7, pull #16
440 USER( ldrt r7, [r1], #4) @ May fault
441 orr r3, r3, r7, push #16
442 str r3, [r0], #4
443 mov ip, r1, lsl #32 - PAGE_SHIFT
444 rsb ip, ip, #0
445 movs ip, ip, lsr #32 - PAGE_SHIFT
446 beq .Lcfu_2fupi
447 cmp r2, ip
448 movlt ip, r2
449 sub r2, r2, ip
450 subs ip, ip, #16
451 blt .Lcfu_2rem8lp
454 .Lcfu_2cpy8lp: mov r3, r7, pull #16
455 ldmia r1!, {r4 - r7} @ Shouldnt fault
456 subs ip, ip, #16
457 orr r3, r3, r4, push #16
458 mov r4, r4, pull #16
459 orr r4, r4, r5, push #16
460 mov r5, r5, pull #16
461 orr r5, r5, r6, push #16
462 mov r6, r6, pull #16
463 orr r6, r6, r7, push #16
464 stmia r0!, {r3 - r6}
465 bpl .Lcfu_2cpy8lp
467 .Lcfu_2rem8lp: tst ip, #8
468 movne r3, r7, pull #16
469 ldmneia r1!, {r4, r7} @ Shouldnt fault
470 orrne r3, r3, r4, push #16
471 movne r4, r4, pull #16
472 orrne r4, r4, r7, push #16
473 stmneia r0!, {r3 - r4}
474 tst ip, #4
475 movne r3, r7, pull #16
476 USER( ldrnet r7, [r1], #4) @ May fault
477 orrne r3, r3, r7, push #16
478 strne r3, [r0], #4
479 ands ip, ip, #3
480 beq .Lcfu_2fupi
481 .Lcfu_2nowords: mov r3, r7, get_byte_2
482 teq ip, #0
483 beq .Lcfu_finished
484 cmp ip, #2
485 strb r3, [r0], #1
486 movge r3, r7, get_byte_3
487 strgeb r3, [r0], #1
488 USER( ldrgtbt r3, [r1], #0) @ May fault
489 strgtb r3, [r0], #1
490 b .Lcfu_finished
492 .Lcfu_3fupi: subs r2, r2, #4
493 addmi ip, r2, #4
494 bmi .Lcfu_3nowords
495 mov r3, r7, pull #24
496 USER( ldrt r7, [r1], #4) @ May fault
497 orr r3, r3, r7, push #8
498 str r3, [r0], #4
499 mov ip, r1, lsl #32 - PAGE_SHIFT
500 rsb ip, ip, #0
501 movs ip, ip, lsr #32 - PAGE_SHIFT
502 beq .Lcfu_3fupi
503 cmp r2, ip
504 movlt ip, r2
505 sub r2, r2, ip
506 subs ip, ip, #16
507 blt .Lcfu_3rem8lp
509 .Lcfu_3cpy8lp: mov r3, r7, pull #24
510 ldmia r1!, {r4 - r7} @ Shouldnt fault
511 orr r3, r3, r4, push #8
512 mov r4, r4, pull #24
513 orr r4, r4, r5, push #8
514 mov r5, r5, pull #24
515 orr r5, r5, r6, push #8
516 mov r6, r6, pull #24
517 orr r6, r6, r7, push #8
518 stmia r0!, {r3 - r6}
519 subs ip, ip, #16
520 bpl .Lcfu_3cpy8lp
522 .Lcfu_3rem8lp: tst ip, #8
523 movne r3, r7, pull #24
524 ldmneia r1!, {r4, r7} @ Shouldnt fault
525 orrne r3, r3, r4, push #8
526 movne r4, r4, pull #24
527 orrne r4, r4, r7, push #8
528 stmneia r0!, {r3 - r4}
529 tst ip, #4
530 movne r3, r7, pull #24
531 USER( ldrnet r7, [r1], #4) @ May fault
532 orrne r3, r3, r7, push #8
533 strne r3, [r0], #4
534 ands ip, ip, #3
535 beq .Lcfu_3fupi
536 .Lcfu_3nowords: mov r3, r7, get_byte_3
537 teq ip, #0
538 beq .Lcfu_finished
539 cmp ip, #2
540 strb r3, [r0], #1
541 USER( ldrgebt r3, [r1], #1) @ May fault
542 strgeb r3, [r0], #1
543 USER( ldrgtbt r3, [r1], #1) @ May fault
544 strgtb r3, [r0], #1
545 b .Lcfu_finished
546 ENDPROC(__copy_from_user)
548 .section .fixup,"ax"
549 .align 0
550 /*
551 * We took an exception. r0 contains a pointer to
552 * the byte not copied.
553 */
554 9001: ldr r2, [sp], #4 @ void *to
555 sub r2, r0, r2 @ bytes copied
556 ldr r1, [sp], #4 @ unsigned long count
557 subs r4, r1, r2 @ bytes left to copy
558 movne r1, r4
559 blne __memzero
560 mov r0, r4
561 ldmfd sp!, {r4 - r7, pc}
562 .previous