| blob | 84ed33ab4c2de8038a0891eb06ddda06e2484a31 |
1 /*
2 * String handling functions for PowerPC.
3 *
4 * Copyright (C) 1996 Paul Mackerras.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11 #include <asm/processor.h>
12 #include <asm/cache.h>
13 #include <asm/errno.h>
14 #include <asm/ppc_asm.h>
16 #define COPY_16_BYTES \
17 lwz r7,4(r4); \
18 lwz r8,8(r4); \
19 lwz r9,12(r4); \
20 lwzu r10,16(r4); \
21 stw r7,4(r6); \
22 stw r8,8(r6); \
23 stw r9,12(r6); \
24 stwu r10,16(r6)
26 #define COPY_16_BYTES_WITHEX(n) \
27 8 ## n ## 0: \
28 lwz r7,4(r4); \
29 8 ## n ## 1: \
30 lwz r8,8(r4); \
31 8 ## n ## 2: \
32 lwz r9,12(r4); \
33 8 ## n ## 3: \
34 lwzu r10,16(r4); \
35 8 ## n ## 4: \
36 stw r7,4(r6); \
37 8 ## n ## 5: \
38 stw r8,8(r6); \
39 8 ## n ## 6: \
40 stw r9,12(r6); \
41 8 ## n ## 7: \
42 stwu r10,16(r6)
44 #define COPY_16_BYTES_EXCODE(n) \
45 9 ## n ## 0: \
46 addi r5,r5,-(16 * n); \
47 b 104f; \
48 9 ## n ## 1: \
49 addi r5,r5,-(16 * n); \
50 b 105f; \
51 .section __ex_table,"a"; \
52 .align 2; \
53 .long 8 ## n ## 0b,9 ## n ## 0b; \
54 .long 8 ## n ## 1b,9 ## n ## 0b; \
55 .long 8 ## n ## 2b,9 ## n ## 0b; \
56 .long 8 ## n ## 3b,9 ## n ## 0b; \
57 .long 8 ## n ## 4b,9 ## n ## 1b; \
58 .long 8 ## n ## 5b,9 ## n ## 1b; \
59 .long 8 ## n ## 6b,9 ## n ## 1b; \
60 .long 8 ## n ## 7b,9 ## n ## 1b; \
61 .text
63 .text
64 .stabs "arch/ppc/lib/",N_SO,0,0,0f
65 .stabs "string.S",N_SO,0,0,0f
67 CACHELINE_BYTES = L1_CACHE_BYTES
68 LG_CACHELINE_BYTES = L1_CACHE_SHIFT
69 CACHELINE_MASK = (L1_CACHE_BYTES-1)
71 _GLOBAL(strcpy)
72 addi r5,r3,-1
73 addi r4,r4,-1
74 1: lbzu r0,1(r4)
75 cmpwi 0,r0,0
76 stbu r0,1(r5)
77 bne 1b
78 blr
80 /* This clears out any unused part of the destination buffer,
81 just as the libc version does. -- paulus */
82 _GLOBAL(strncpy)
83 cmpwi 0,r5,0
84 beqlr
85 mtctr r5
86 addi r6,r3,-1
87 addi r4,r4,-1
88 1: lbzu r0,1(r4)
89 cmpwi 0,r0,0
90 stbu r0,1(r6)
91 bdnzf 2,1b /* dec ctr, branch if ctr != 0 && !cr0.eq */
92 bnelr /* if we didn't hit a null char, we're done */
93 mfctr r5
94 cmpwi 0,r5,0 /* any space left in destination buffer? */
95 beqlr /* we know r0 == 0 here */
96 2: stbu r0,1(r6) /* clear it out if so */
97 bdnz 2b
98 blr
100 _GLOBAL(strcat)
101 addi r5,r3,-1
102 addi r4,r4,-1
103 1: lbzu r0,1(r5)
104 cmpwi 0,r0,0
105 bne 1b
106 addi r5,r5,-1
107 1: lbzu r0,1(r4)
108 cmpwi 0,r0,0
109 stbu r0,1(r5)
110 bne 1b
111 blr
113 _GLOBAL(strcmp)
114 addi r5,r3,-1
115 addi r4,r4,-1
116 1: lbzu r3,1(r5)
117 cmpwi 1,r3,0
118 lbzu r0,1(r4)
119 subf. r3,r0,r3
120 beqlr 1
121 beq 1b
122 blr
124 _GLOBAL(strlen)
125 addi r4,r3,-1
126 1: lbzu r0,1(r4)
127 cmpwi 0,r0,0
128 bne 1b
129 subf r3,r3,r4
130 blr
132 /*
133 * Use dcbz on the complete cache lines in the destination
134 * to set them to zero. This requires that the destination
135 * area is cacheable. -- paulus
136 */
137 _GLOBAL(cacheable_memzero)
138 mr r5,r4
139 li r4,0
140 addi r6,r3,-4
141 cmplwi 0,r5,4
142 blt 7f
143 stwu r4,4(r6)
144 beqlr
145 andi. r0,r6,3
146 add r5,r0,r5
147 subf r6,r0,r6
148 clrlwi r7,r6,32-LG_CACHELINE_BYTES
149 add r8,r7,r5
150 srwi r9,r8,LG_CACHELINE_BYTES
151 addic. r9,r9,-1 /* total number of complete cachelines */
152 ble 2f
153 xori r0,r7,CACHELINE_MASK & ~3
154 srwi. r0,r0,2
155 beq 3f
156 mtctr r0
157 4: stwu r4,4(r6)
158 bdnz 4b
159 3: mtctr r9
160 li r7,4
161 #if !defined(CONFIG_8xx)
162 10: dcbz r7,r6
163 #else
164 10: stw r4, 4(r6)
165 stw r4, 8(r6)
166 stw r4, 12(r6)
167 stw r4, 16(r6)
168 #if CACHE_LINE_SIZE >= 32
169 stw r4, 20(r6)
170 stw r4, 24(r6)
171 stw r4, 28(r6)
172 stw r4, 32(r6)
173 #endif /* CACHE_LINE_SIZE */
174 #endif
175 addi r6,r6,CACHELINE_BYTES
176 bdnz 10b
177 clrlwi r5,r8,32-LG_CACHELINE_BYTES
178 addi r5,r5,4
179 2: srwi r0,r5,2
180 mtctr r0
181 bdz 6f
182 1: stwu r4,4(r6)
183 bdnz 1b
184 6: andi. r5,r5,3
185 7: cmpwi 0,r5,0
186 beqlr
187 mtctr r5
188 addi r6,r6,3
189 8: stbu r4,1(r6)
190 bdnz 8b
191 blr
193 _GLOBAL(memset)
194 rlwimi r4,r4,8,16,23
195 rlwimi r4,r4,16,0,15
196 addi r6,r3,-4
197 cmplwi 0,r5,4
198 blt 7f
199 stwu r4,4(r6)
200 beqlr
201 andi. r0,r6,3
202 add r5,r0,r5
203 subf r6,r0,r6
204 srwi r0,r5,2
205 mtctr r0
206 bdz 6f
207 1: stwu r4,4(r6)
208 bdnz 1b
209 6: andi. r5,r5,3
210 7: cmpwi 0,r5,0
211 beqlr
212 mtctr r5
213 addi r6,r6,3
214 8: stbu r4,1(r6)
215 bdnz 8b
216 blr
218 /*
219 * This version uses dcbz on the complete cache lines in the
220 * destination area to reduce memory traffic. This requires that
221 * the destination area is cacheable.
222 * We only use this version if the source and dest don't overlap.
223 * -- paulus.
224 */
225 _GLOBAL(cacheable_memcpy)
226 add r7,r3,r5 /* test if the src & dst overlap */
227 add r8,r4,r5
228 cmplw 0,r4,r7
229 cmplw 1,r3,r8
230 crand 0,0,4 /* cr0.lt &= cr1.lt */
231 blt memcpy /* if regions overlap */
233 addi r4,r4,-4
234 addi r6,r3,-4
235 neg r0,r3
236 andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */
237 beq 58f
239 cmplw 0,r5,r0 /* is this more than total to do? */
240 blt 63f /* if not much to do */
241 andi. r8,r0,3 /* get it word-aligned first */
242 subf r5,r0,r5
243 mtctr r8
244 beq+ 61f
245 70: lbz r9,4(r4) /* do some bytes */
246 stb r9,4(r6)
247 addi r4,r4,1
248 addi r6,r6,1
249 bdnz 70b
250 61: srwi. r0,r0,2
251 mtctr r0
252 beq 58f
253 72: lwzu r9,4(r4) /* do some words */
254 stwu r9,4(r6)
255 bdnz 72b
257 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
258 clrlwi r5,r5,32-LG_CACHELINE_BYTES
259 li r11,4
260 mtctr r0
261 beq 63f
262 53:
263 #if !defined(CONFIG_8xx)
264 dcbz r11,r6
265 #endif
266 COPY_16_BYTES
267 #if L1_CACHE_BYTES >= 32
268 COPY_16_BYTES
269 #if L1_CACHE_BYTES >= 64
270 COPY_16_BYTES
271 COPY_16_BYTES
272 #if L1_CACHE_BYTES >= 128
273 COPY_16_BYTES
274 COPY_16_BYTES
275 COPY_16_BYTES
276 COPY_16_BYTES
277 #endif
278 #endif
279 #endif
280 bdnz 53b
282 63: srwi. r0,r5,2
283 mtctr r0
284 beq 64f
285 30: lwzu r0,4(r4)
286 stwu r0,4(r6)
287 bdnz 30b
289 64: andi. r0,r5,3
290 mtctr r0
291 beq+ 65f
292 40: lbz r0,4(r4)
293 stb r0,4(r6)
294 addi r4,r4,1
295 addi r6,r6,1
296 bdnz 40b
297 65: blr
299 _GLOBAL(memmove)
300 cmplw 0,r3,r4
301 bgt backwards_memcpy
302 /* fall through */
304 _GLOBAL(memcpy)
305 srwi. r7,r5,3
306 addi r6,r3,-4
307 addi r4,r4,-4
308 beq 2f /* if less than 8 bytes to do */
309 andi. r0,r6,3 /* get dest word aligned */
310 mtctr r7
311 bne 5f
312 1: lwz r7,4(r4)
313 lwzu r8,8(r4)
314 stw r7,4(r6)
315 stwu r8,8(r6)
316 bdnz 1b
317 andi. r5,r5,7
318 2: cmplwi 0,r5,4
319 blt 3f
320 lwzu r0,4(r4)
321 addi r5,r5,-4
322 stwu r0,4(r6)
323 3: cmpwi 0,r5,0
324 beqlr
325 mtctr r5
326 addi r4,r4,3
327 addi r6,r6,3
328 4: lbzu r0,1(r4)
329 stbu r0,1(r6)
330 bdnz 4b
331 blr
332 5: subfic r0,r0,4
333 mtctr r0
334 6: lbz r7,4(r4)
335 addi r4,r4,1
336 stb r7,4(r6)
337 addi r6,r6,1
338 bdnz 6b
339 subf r5,r0,r5
340 rlwinm. r7,r5,32-3,3,31
341 beq 2b
342 mtctr r7
343 b 1b
345 _GLOBAL(backwards_memcpy)
346 rlwinm. r7,r5,32-3,3,31 /* r0 = r5 >> 3 */
347 add r6,r3,r5
348 add r4,r4,r5
349 beq 2f
350 andi. r0,r6,3
351 mtctr r7
352 bne 5f
353 1: lwz r7,-4(r4)
354 lwzu r8,-8(r4)
355 stw r7,-4(r6)
356 stwu r8,-8(r6)
357 bdnz 1b
358 andi. r5,r5,7
359 2: cmplwi 0,r5,4
360 blt 3f
361 lwzu r0,-4(r4)
362 subi r5,r5,4
363 stwu r0,-4(r6)
364 3: cmpwi 0,r5,0
365 beqlr
366 mtctr r5
367 4: lbzu r0,-1(r4)
368 stbu r0,-1(r6)
369 bdnz 4b
370 blr
371 5: mtctr r0
372 6: lbzu r7,-1(r4)
373 stbu r7,-1(r6)
374 bdnz 6b
375 subf r5,r0,r5
376 rlwinm. r7,r5,32-3,3,31
377 beq 2b
378 mtctr r7
379 b 1b
381 _GLOBAL(memcmp)
382 cmpwi 0,r5,0
383 ble- 2f
384 mtctr r5
385 addi r6,r3,-1
386 addi r4,r4,-1
387 1: lbzu r3,1(r6)
388 lbzu r0,1(r4)
389 subf. r3,r0,r3
390 bdnzt 2,1b
391 blr
392 2: li r3,0
393 blr
395 _GLOBAL(memchr)
396 cmpwi 0,r5,0
397 ble- 2f
398 mtctr r5
399 addi r3,r3,-1
400 1: lbzu r0,1(r3)
401 cmpw 0,r0,r4
402 bdnzf 2,1b
403 beqlr
404 2: li r3,0
405 blr
407 _GLOBAL(__copy_tofrom_user)
408 addi r4,r4,-4
409 addi r6,r3,-4
410 neg r0,r3
411 andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */
412 beq 58f
414 cmplw 0,r5,r0 /* is this more than total to do? */
415 blt 63f /* if not much to do */
416 andi. r8,r0,3 /* get it word-aligned first */
417 mtctr r8
418 beq+ 61f
419 70: lbz r9,4(r4) /* do some bytes */
420 71: stb r9,4(r6)
421 addi r4,r4,1
422 addi r6,r6,1
423 bdnz 70b
424 61: subf r5,r0,r5
425 srwi. r0,r0,2
426 mtctr r0
427 beq 58f
428 72: lwzu r9,4(r4) /* do some words */
429 73: stwu r9,4(r6)
430 bdnz 72b
432 .section __ex_table,"a"
433 .align 2
434 .long 70b,100f
435 .long 71b,101f
436 .long 72b,102f
437 .long 73b,103f
438 .text
440 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
441 clrlwi r5,r5,32-LG_CACHELINE_BYTES
442 li r11,4
443 beq 63f
445 #ifdef CONFIG_8xx
446 /* Don't use prefetch on 8xx */
447 mtctr r0
448 li r0,0
449 53: COPY_16_BYTES_WITHEX(0)
450 bdnz 53b
452 #else /* not CONFIG_8xx */
453 /* Here we decide how far ahead to prefetch the source */
454 li r3,4
455 cmpwi r0,1
456 li r7,0
457 ble 114f
458 li r7,1
459 #if MAX_COPY_PREFETCH > 1
460 /* Heuristically, for large transfers we prefetch
461 MAX_COPY_PREFETCH cachelines ahead. For small transfers
462 we prefetch 1 cacheline ahead. */
463 cmpwi r0,MAX_COPY_PREFETCH
464 ble 112f
465 li r7,MAX_COPY_PREFETCH
466 112: mtctr r7
467 111: dcbt r3,r4
468 addi r3,r3,CACHELINE_BYTES
469 bdnz 111b
470 #else
471 dcbt r3,r4
472 addi r3,r3,CACHELINE_BYTES
473 #endif /* MAX_COPY_PREFETCH > 1 */
475 114: subf r8,r7,r0
476 mr r0,r7
477 mtctr r8
479 53: dcbt r3,r4
480 54: dcbz r11,r6
481 .section __ex_table,"a"
482 .align 2
483 .long 54b,105f
484 .text
485 /* the main body of the cacheline loop */
486 COPY_16_BYTES_WITHEX(0)
487 #if L1_CACHE_BYTES >= 32
488 COPY_16_BYTES_WITHEX(1)
489 #if L1_CACHE_BYTES >= 64
490 COPY_16_BYTES_WITHEX(2)
491 COPY_16_BYTES_WITHEX(3)
492 #if L1_CACHE_BYTES >= 128
493 COPY_16_BYTES_WITHEX(4)
494 COPY_16_BYTES_WITHEX(5)
495 COPY_16_BYTES_WITHEX(6)
496 COPY_16_BYTES_WITHEX(7)
497 #endif
498 #endif
499 #endif
500 bdnz 53b
501 cmpwi r0,0
502 li r3,4
503 li r7,0
504 bne 114b
505 #endif /* CONFIG_8xx */
507 63: srwi. r0,r5,2
508 mtctr r0
509 beq 64f
510 30: lwzu r0,4(r4)
511 31: stwu r0,4(r6)
512 bdnz 30b
514 64: andi. r0,r5,3
515 mtctr r0
516 beq+ 65f
517 40: lbz r0,4(r4)
518 41: stb r0,4(r6)
519 addi r4,r4,1
520 addi r6,r6,1
521 bdnz 40b
522 65: li r3,0
523 blr
525 /* read fault, initial single-byte copy */
526 100: li r9,0
527 b 90f
528 /* write fault, initial single-byte copy */
529 101: li r9,1
530 90: subf r5,r8,r5
531 li r3,0
532 b 99f
533 /* read fault, initial word copy */
534 102: li r9,0
535 b 91f
536 /* write fault, initial word copy */
537 103: li r9,1
538 91: li r3,2
539 b 99f
541 /*
542 * this stuff handles faults in the cacheline loop and branches to either
543 * 104f (if in read part) or 105f (if in write part), after updating r5
544 */
545 COPY_16_BYTES_EXCODE(0)
546 #if L1_CACHE_BYTES >= 32
547 COPY_16_BYTES_EXCODE(1)
548 #if L1_CACHE_BYTES >= 64
549 COPY_16_BYTES_EXCODE(2)
550 COPY_16_BYTES_EXCODE(3)
551 #if L1_CACHE_BYTES >= 128
552 COPY_16_BYTES_EXCODE(4)
553 COPY_16_BYTES_EXCODE(5)
554 COPY_16_BYTES_EXCODE(6)
555 COPY_16_BYTES_EXCODE(7)
556 #endif
557 #endif
558 #endif
560 /* read fault in cacheline loop */
561 104: li r9,0
562 b 92f
563 /* fault on dcbz (effectively a write fault) */
564 /* or write fault in cacheline loop */
565 105: li r9,1
566 92: li r3,LG_CACHELINE_BYTES
567 mfctr r8
568 add r0,r0,r8
569 b 106f
570 /* read fault in final word loop */
571 108: li r9,0
572 b 93f
573 /* write fault in final word loop */
574 109: li r9,1
575 93: andi. r5,r5,3
576 li r3,2
577 b 99f
578 /* read fault in final byte loop */
579 110: li r9,0
580 b 94f
581 /* write fault in final byte loop */
582 111: li r9,1
583 94: li r5,0
584 li r3,0
585 /*
586 * At this stage the number of bytes not copied is
587 * r5 + (ctr << r3), and r9 is 0 for read or 1 for write.
588 */
589 99: mfctr r0
590 106: slw r3,r0,r3
591 add. r3,r3,r5
592 beq 120f /* shouldn't happen */
593 cmpwi 0,r9,0
594 bne 120f
595 /* for a read fault, first try to continue the copy one byte at a time */
596 mtctr r3
597 130: lbz r0,4(r4)
598 131: stb r0,4(r6)
599 addi r4,r4,1
600 addi r6,r6,1
601 bdnz 130b
602 /* then clear out the destination: r3 bytes starting at 4(r6) */
603 132: mfctr r3
604 srwi. r0,r3,2
605 li r9,0
606 mtctr r0
607 beq 113f
608 112: stwu r9,4(r6)
609 bdnz 112b
610 113: andi. r0,r3,3
611 mtctr r0
612 beq 120f
613 114: stb r9,4(r6)
614 addi r6,r6,1
615 bdnz 114b
616 120: blr
618 .section __ex_table,"a"
619 .align 2
620 .long 30b,108b
621 .long 31b,109b
622 .long 40b,110b
623 .long 41b,111b
624 .long 130b,132b
625 .long 131b,120b
626 .long 112b,120b
627 .long 114b,120b
628 .text
630 _GLOBAL(__clear_user)
631 addi r6,r3,-4
632 li r3,0
633 li r5,0
634 cmplwi 0,r4,4
635 blt 7f
636 /* clear a single word */
637 11: stwu r5,4(r6)
638 beqlr
639 /* clear word sized chunks */
640 andi. r0,r6,3
641 add r4,r0,r4
642 subf r6,r0,r6
643 srwi r0,r4,2
644 andi. r4,r4,3
645 mtctr r0
646 bdz 7f
647 1: stwu r5,4(r6)
648 bdnz 1b
649 /* clear byte sized chunks */
650 7: cmpwi 0,r4,0
651 beqlr
652 mtctr r4
653 addi r6,r6,3
654 8: stbu r5,1(r6)
655 bdnz 8b
656 blr
657 90: mr r3,r4
658 blr
659 91: mfctr r3
660 slwi r3,r3,2
661 add r3,r3,r4
662 blr
663 92: mfctr r3
664 blr
666 .section __ex_table,"a"
667 .align 2
668 .long 11b,90b
669 .long 1b,91b
670 .long 8b,92b
671 .text
673 _GLOBAL(__strncpy_from_user)
674 addi r6,r3,-1
675 addi r4,r4,-1
676 cmpwi 0,r5,0
677 beq 2f
678 mtctr r5
679 1: lbzu r0,1(r4)
680 cmpwi 0,r0,0
681 stbu r0,1(r6)
682 bdnzf 2,1b /* dec ctr, branch if ctr != 0 && !cr0.eq */
683 beq 3f
684 2: addi r6,r6,1
685 3: subf r3,r3,r6
686 blr
687 99: li r3,-EFAULT
688 blr
690 .section __ex_table,"a"
691 .align 2
692 .long 1b,99b
693 .text
695 /* r3 = str, r4 = len (> 0), r5 = top (highest addr) */
696 _GLOBAL(__strnlen_user)
697 addi r7,r3,-1
698 subf r6,r7,r5 /* top+1 - str */
699 cmplw 0,r4,r6
700 bge 0f
701 mr r6,r4
702 0: mtctr r6 /* ctr = min(len, top - str) */
703 1: lbzu r0,1(r7) /* get next byte */
704 cmpwi 0,r0,0
705 bdnzf 2,1b /* loop if --ctr != 0 && byte != 0 */
706 addi r7,r7,1
707 subf r3,r3,r7 /* number of bytes we have looked at */
708 beqlr /* return if we found a 0 byte */
709 cmpw 0,r3,r4 /* did we look at all len bytes? */
710 blt 99f /* if not, must have hit top */
711 addi r3,r4,1 /* return len + 1 to indicate no null found */
712 blr
713 99: li r3,0 /* bad address, return 0 */
714 blr
716 .section __ex_table,"a"
717 .align 2
718 .long 1b,99b