| blob | f681556c6b86d99525c3674daa071ff4d5c3e6d5 |
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 *
4 * Optimized version of the copy_user() routine.
5 * It is used to copy date across the kernel/user boundary.
6 *
7 * The source and destination are always on opposite side of
8 * the boundary. When reading from user space we must catch
9 * faults on loads. When writing to user space we must catch
10 * errors on stores. Note that because of the nature of the copy
11 * we don't need to worry about overlapping regions.
12 *
13 *
14 * Inputs:
15 * in0 address of source buffer
16 * in1 address of destination buffer
17 * in2 number of bytes to copy
18 *
19 * Outputs:
20 * ret0 0 in case of success. The number of bytes NOT copied in
21 * case of error.
22 *
23 * Copyright (C) 2000-2001 Hewlett-Packard Co
24 * Stephane Eranian <eranian@hpl.hp.com>
25 *
26 * Fixme:
27 * - handle the case where we have more than 16 bytes and the alignment
28 * are different.
29 * - more benchmarking
30 * - fix extraneous stop bit introduced by the EX() macro.
31 */
33 #include <asm/asmmacro.h>
34 #include <asm/export.h>
36 //
37 // Tuneable parameters
38 //
39 #define COPY_BREAK 16 // we do byte copy below (must be >=16)
40 #define PIPE_DEPTH 21 // pipe depth
42 #define EPI p[PIPE_DEPTH-1]
44 //
45 // arguments
46 //
47 #define dst in0
48 #define src in1
49 #define len in2
51 //
52 // local registers
53 //
54 #define t1 r2 // rshift in bytes
55 #define t2 r3 // lshift in bytes
56 #define rshift r14 // right shift in bits
57 #define lshift r15 // left shift in bits
58 #define word1 r16
59 #define word2 r17
60 #define cnt r18
61 #define len2 r19
62 #define saved_lc r20
63 #define saved_pr r21
64 #define tmp r22
65 #define val r23
66 #define src1 r24
67 #define dst1 r25
68 #define src2 r26
69 #define dst2 r27
70 #define len1 r28
71 #define enddst r29
72 #define endsrc r30
73 #define saved_pfs r31
75 GLOBAL_ENTRY(__copy_user)
76 .prologue
77 .save ar.pfs, saved_pfs
78 alloc saved_pfs=ar.pfs,3,((2*PIPE_DEPTH+7)&~7),0,((2*PIPE_DEPTH+7)&~7)
80 .rotr val1[PIPE_DEPTH],val2[PIPE_DEPTH]
81 .rotp p[PIPE_DEPTH]
83 adds len2=-1,len // br.ctop is repeat/until
84 mov ret0=r0
86 ;; // RAW of cfm when len=0
87 cmp.eq p8,p0=r0,len // check for zero length
88 .save ar.lc, saved_lc
89 mov saved_lc=ar.lc // preserve ar.lc (slow)
90 (p8) br.ret.spnt.many rp // empty mempcy()
91 ;;
92 add enddst=dst,len // first byte after end of source
93 add endsrc=src,len // first byte after end of destination
94 .save pr, saved_pr
95 mov saved_pr=pr // preserve predicates
97 .body
99 mov dst1=dst // copy because of rotation
100 mov ar.ec=PIPE_DEPTH
101 mov pr.rot=1<<16 // p16=true all others are false
103 mov src1=src // copy because of rotation
104 mov ar.lc=len2 // initialize lc for small count
105 cmp.lt p10,p7=COPY_BREAK,len // if len > COPY_BREAK then long copy
107 xor tmp=src,dst // same alignment test prepare
108 (p10) br.cond.dptk .long_copy_user
109 ;; // RAW pr.rot/p16 ?
110 //
111 // Now we do the byte by byte loop with software pipeline
112 //
113 // p7 is necessarily false by now
114 1:
115 EX(.failure_in_pipe1,(p16) ld1 val1[0]=[src1],1)
116 EX(.failure_out,(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1)
117 br.ctop.dptk.few 1b
118 ;;
119 mov ar.lc=saved_lc
120 mov pr=saved_pr,0xffffffffffff0000
121 mov ar.pfs=saved_pfs // restore ar.ec
122 br.ret.sptk.many rp // end of short memcpy
124 //
125 // Not 8-byte aligned
126 //
127 .diff_align_copy_user:
128 // At this point we know we have more than 16 bytes to copy
129 // and also that src and dest do _not_ have the same alignment.
130 and src2=0x7,src1 // src offset
131 and dst2=0x7,dst1 // dst offset
132 ;;
133 // The basic idea is that we copy byte-by-byte at the head so
134 // that we can reach 8-byte alignment for both src1 and dst1.
135 // Then copy the body using software pipelined 8-byte copy,
136 // shifting the two back-to-back words right and left, then copy
137 // the tail by copying byte-by-byte.
138 //
139 // Fault handling. If the byte-by-byte at the head fails on the
140 // load, then restart and finish the pipleline by copying zeros
141 // to the dst1. Then copy zeros for the rest of dst1.
142 // If 8-byte software pipeline fails on the load, do the same as
143 // failure_in3 does. If the byte-by-byte at the tail fails, it is
144 // handled simply by failure_in_pipe1.
145 //
146 // The case p14 represents the source has more bytes in the
147 // the first word (by the shifted part), whereas the p15 needs to
148 // copy some bytes from the 2nd word of the source that has the
149 // tail of the 1st of the destination.
150 //
152 //
153 // Optimization. If dst1 is 8-byte aligned (quite common), we don't need
154 // to copy the head to dst1, to start 8-byte copy software pipeline.
155 // We know src1 is not 8-byte aligned in this case.
156 //
157 cmp.eq p14,p15=r0,dst2
158 (p15) br.cond.spnt 1f
159 ;;
160 sub t1=8,src2
161 mov t2=src2
162 ;;
163 shl rshift=t2,3
164 sub len1=len,t1 // set len1
165 ;;
166 sub lshift=64,rshift
167 ;;
168 br.cond.spnt .word_copy_user
169 ;;
170 1:
171 cmp.leu p14,p15=src2,dst2
172 sub t1=dst2,src2
173 ;;
174 .pred.rel "mutex", p14, p15
175 (p14) sub word1=8,src2 // (8 - src offset)
176 (p15) sub t1=r0,t1 // absolute value
177 (p15) sub word1=8,dst2 // (8 - dst offset)
178 ;;
179 // For the case p14, we don't need to copy the shifted part to
180 // the 1st word of destination.
181 sub t2=8,t1
182 (p14) sub word1=word1,t1
183 ;;
184 sub len1=len,word1 // resulting len
185 (p15) shl rshift=t1,3 // in bits
186 (p14) shl rshift=t2,3
187 ;;
188 (p14) sub len1=len1,t1
189 adds cnt=-1,word1
190 ;;
191 sub lshift=64,rshift
192 mov ar.ec=PIPE_DEPTH
193 mov pr.rot=1<<16 // p16=true all others are false
194 mov ar.lc=cnt
195 ;;
196 2:
197 EX(.failure_in_pipe2,(p16) ld1 val1[0]=[src1],1)
198 EX(.failure_out,(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1)
199 br.ctop.dptk.few 2b
200 ;;
201 clrrrb
202 ;;
203 .word_copy_user:
204 cmp.gtu p9,p0=16,len1
205 (p9) br.cond.spnt 4f // if (16 > len1) skip 8-byte copy
206 ;;
207 shr.u cnt=len1,3 // number of 64-bit words
208 ;;
209 adds cnt=-1,cnt
210 ;;
211 .pred.rel "mutex", p14, p15
212 (p14) sub src1=src1,t2
213 (p15) sub src1=src1,t1
214 //
215 // Now both src1 and dst1 point to an 8-byte aligned address. And
216 // we have more than 8 bytes to copy.
217 //
218 mov ar.lc=cnt
219 mov ar.ec=PIPE_DEPTH
220 mov pr.rot=1<<16 // p16=true all others are false
221 ;;
222 3:
223 //
224 // The pipleline consists of 3 stages:
225 // 1 (p16): Load a word from src1
226 // 2 (EPI_1): Shift right pair, saving to tmp
227 // 3 (EPI): Store tmp to dst1
228 //
229 // To make it simple, use at least 2 (p16) loops to set up val1[n]
230 // because we need 2 back-to-back val1[] to get tmp.
231 // Note that this implies EPI_2 must be p18 or greater.
232 //
234 #define EPI_1 p[PIPE_DEPTH-2]
235 #define SWITCH(pred, shift) cmp.eq pred,p0=shift,rshift
236 #define CASE(pred, shift) \
237 (pred) br.cond.spnt .copy_user_bit##shift
238 #define BODY(rshift) \
239 .copy_user_bit##rshift: \
240 1: \
241 EX(.failure_out,(EPI) st8 [dst1]=tmp,8); \
242 (EPI_1) shrp tmp=val1[PIPE_DEPTH-2],val1[PIPE_DEPTH-1],rshift; \
243 EX(3f,(p16) ld8 val1[1]=[src1],8); \
244 (p16) mov val1[0]=r0; \
245 br.ctop.dptk 1b; \
246 ;; \
247 br.cond.sptk.many .diff_align_do_tail; \
248 2: \
249 (EPI) st8 [dst1]=tmp,8; \
250 (EPI_1) shrp tmp=val1[PIPE_DEPTH-2],val1[PIPE_DEPTH-1],rshift; \
251 3: \
252 (p16) mov val1[1]=r0; \
253 (p16) mov val1[0]=r0; \
254 br.ctop.dptk 2b; \
255 ;; \
256 br.cond.sptk.many .failure_in2
258 //
259 // Since the instruction 'shrp' requires a fixed 128-bit value
260 // specifying the bits to shift, we need to provide 7 cases
261 // below.
262 //
263 SWITCH(p6, 8)
264 SWITCH(p7, 16)
265 SWITCH(p8, 24)
266 SWITCH(p9, 32)
267 SWITCH(p10, 40)
268 SWITCH(p11, 48)
269 SWITCH(p12, 56)
270 ;;
271 CASE(p6, 8)
272 CASE(p7, 16)
273 CASE(p8, 24)
274 CASE(p9, 32)
275 CASE(p10, 40)
276 CASE(p11, 48)
277 CASE(p12, 56)
278 ;;
279 BODY(8)
280 BODY(16)
281 BODY(24)
282 BODY(32)
283 BODY(40)
284 BODY(48)
285 BODY(56)
286 ;;
287 .diff_align_do_tail:
288 .pred.rel "mutex", p14, p15
289 (p14) sub src1=src1,t1
290 (p14) adds dst1=-8,dst1
291 (p15) sub dst1=dst1,t1
292 ;;
293 4:
294 // Tail correction.
295 //
296 // The problem with this piplelined loop is that the last word is not
297 // loaded and thus parf of the last word written is not correct.
298 // To fix that, we simply copy the tail byte by byte.
300 sub len1=endsrc,src1,1
301 clrrrb
302 ;;
303 mov ar.ec=PIPE_DEPTH
304 mov pr.rot=1<<16 // p16=true all others are false
305 mov ar.lc=len1
306 ;;
307 5:
308 EX(.failure_in_pipe1,(p16) ld1 val1[0]=[src1],1)
309 EX(.failure_out,(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1)
310 br.ctop.dptk.few 5b
311 ;;
312 mov ar.lc=saved_lc
313 mov pr=saved_pr,0xffffffffffff0000
314 mov ar.pfs=saved_pfs
315 br.ret.sptk.many rp
317 //
318 // Beginning of long mempcy (i.e. > 16 bytes)
319 //
320 .long_copy_user:
321 tbit.nz p6,p7=src1,0 // odd alignment
322 and tmp=7,tmp
323 ;;
324 cmp.eq p10,p8=r0,tmp
325 mov len1=len // copy because of rotation
326 (p8) br.cond.dpnt .diff_align_copy_user
327 ;;
328 // At this point we know we have more than 16 bytes to copy
329 // and also that both src and dest have the same alignment
330 // which may not be the one we want. So for now we must move
331 // forward slowly until we reach 16byte alignment: no need to
332 // worry about reaching the end of buffer.
333 //
334 EX(.failure_in1,(p6) ld1 val1[0]=[src1],1) // 1-byte aligned
335 (p6) adds len1=-1,len1;;
336 tbit.nz p7,p0=src1,1
337 ;;
338 EX(.failure_in1,(p7) ld2 val1[1]=[src1],2) // 2-byte aligned
339 (p7) adds len1=-2,len1;;
340 tbit.nz p8,p0=src1,2
341 ;;
342 //
343 // Stop bit not required after ld4 because if we fail on ld4
344 // we have never executed the ld1, therefore st1 is not executed.
345 //
346 EX(.failure_in1,(p8) ld4 val2[0]=[src1],4) // 4-byte aligned
347 ;;
348 EX(.failure_out,(p6) st1 [dst1]=val1[0],1)
349 tbit.nz p9,p0=src1,3
350 ;;
351 //
352 // Stop bit not required after ld8 because if we fail on ld8
353 // we have never executed the ld2, therefore st2 is not executed.
354 //
355 EX(.failure_in1,(p9) ld8 val2[1]=[src1],8) // 8-byte aligned
356 EX(.failure_out,(p7) st2 [dst1]=val1[1],2)
357 (p8) adds len1=-4,len1
358 ;;
359 EX(.failure_out, (p8) st4 [dst1]=val2[0],4)
360 (p9) adds len1=-8,len1;;
361 shr.u cnt=len1,4 // number of 128-bit (2x64bit) words
362 ;;
363 EX(.failure_out, (p9) st8 [dst1]=val2[1],8)
364 tbit.nz p6,p0=len1,3
365 cmp.eq p7,p0=r0,cnt
366 adds tmp=-1,cnt // br.ctop is repeat/until
367 (p7) br.cond.dpnt .dotail // we have less than 16 bytes left
368 ;;
369 adds src2=8,src1
370 adds dst2=8,dst1
371 mov ar.lc=tmp
372 ;;
373 //
374 // 16bytes/iteration
375 //
376 2:
377 EX(.failure_in3,(p16) ld8 val1[0]=[src1],16)
378 (p16) ld8 val2[0]=[src2],16
380 EX(.failure_out, (EPI) st8 [dst1]=val1[PIPE_DEPTH-1],16)
381 (EPI) st8 [dst2]=val2[PIPE_DEPTH-1],16
382 br.ctop.dptk 2b
383 ;; // RAW on src1 when fall through from loop
384 //
385 // Tail correction based on len only
386 //
387 // No matter where we come from (loop or test) the src1 pointer
388 // is 16 byte aligned AND we have less than 16 bytes to copy.
389 //
390 .dotail:
391 EX(.failure_in1,(p6) ld8 val1[0]=[src1],8) // at least 8 bytes
392 tbit.nz p7,p0=len1,2
393 ;;
394 EX(.failure_in1,(p7) ld4 val1[1]=[src1],4) // at least 4 bytes
395 tbit.nz p8,p0=len1,1
396 ;;
397 EX(.failure_in1,(p8) ld2 val2[0]=[src1],2) // at least 2 bytes
398 tbit.nz p9,p0=len1,0
399 ;;
400 EX(.failure_out, (p6) st8 [dst1]=val1[0],8)
401 ;;
402 EX(.failure_in1,(p9) ld1 val2[1]=[src1]) // only 1 byte left
403 mov ar.lc=saved_lc
404 ;;
405 EX(.failure_out,(p7) st4 [dst1]=val1[1],4)
406 mov pr=saved_pr,0xffffffffffff0000
407 ;;
408 EX(.failure_out, (p8) st2 [dst1]=val2[0],2)
409 mov ar.pfs=saved_pfs
410 ;;
411 EX(.failure_out, (p9) st1 [dst1]=val2[1])
412 br.ret.sptk.many rp
415 //
416 // Here we handle the case where the byte by byte copy fails
417 // on the load.
418 // Several factors make the zeroing of the rest of the buffer kind of
419 // tricky:
420 // - the pipeline: loads/stores are not in sync (pipeline)
421 //
422 // In the same loop iteration, the dst1 pointer does not directly
423 // reflect where the faulty load was.
424 //
425 // - pipeline effect
426 // When you get a fault on load, you may have valid data from
427 // previous loads not yet store in transit. Such data must be
428 // store normally before moving onto zeroing the rest.
429 //
430 // - single/multi dispersal independence.
431 //
432 // solution:
433 // - we don't disrupt the pipeline, i.e. data in transit in
434 // the software pipeline will be eventually move to memory.
435 // We simply replace the load with a simple mov and keep the
436 // pipeline going. We can't really do this inline because
437 // p16 is always reset to 1 when lc > 0.
438 //
439 .failure_in_pipe1:
440 sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied
441 1:
442 (p16) mov val1[0]=r0
443 (EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1
444 br.ctop.dptk 1b
445 ;;
446 mov pr=saved_pr,0xffffffffffff0000
447 mov ar.lc=saved_lc
448 mov ar.pfs=saved_pfs
449 br.ret.sptk.many rp
451 //
452 // This is the case where the byte by byte copy fails on the load
453 // when we copy the head. We need to finish the pipeline and copy
454 // zeros for the rest of the destination. Since this happens
455 // at the top we still need to fill the body and tail.
456 .failure_in_pipe2:
457 sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied
458 2:
459 (p16) mov val1[0]=r0
460 (EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1
461 br.ctop.dptk 2b
462 ;;
463 sub len=enddst,dst1,1 // precompute len
464 br.cond.dptk.many .failure_in1bis
465 ;;
467 //
468 // Here we handle the head & tail part when we check for alignment.
469 // The following code handles only the load failures. The
470 // main diffculty comes from the fact that loads/stores are
471 // scheduled. So when you fail on a load, the stores corresponding
472 // to previous successful loads must be executed.
473 //
474 // However some simplifications are possible given the way
475 // things work.
476 //
477 // 1) HEAD
478 // Theory of operation:
479 //
480 // Page A | Page B
481 // ---------|-----
482 // 1|8 x
483 // 1 2|8 x
484 // 4|8 x
485 // 1 4|8 x
486 // 2 4|8 x
487 // 1 2 4|8 x
488 // |1
489 // |2 x
490 // |4 x
491 //
492 // page_size >= 4k (2^12). (x means 4, 2, 1)
493 // Here we suppose Page A exists and Page B does not.
494 //
495 // As we move towards eight byte alignment we may encounter faults.
496 // The numbers on each page show the size of the load (current alignment).
497 //
498 // Key point:
499 // - if you fail on 1, 2, 4 then you have never executed any smaller
500 // size loads, e.g. failing ld4 means no ld1 nor ld2 executed
501 // before.
502 //
503 // This allows us to simplify the cleanup code, because basically you
504 // only have to worry about "pending" stores in the case of a failing
505 // ld8(). Given the way the code is written today, this means only
506 // worry about st2, st4. There we can use the information encapsulated
507 // into the predicates.
508 //
509 // Other key point:
510 // - if you fail on the ld8 in the head, it means you went straight
511 // to it, i.e. 8byte alignment within an unexisting page.
512 // Again this comes from the fact that if you crossed just for the ld8 then
513 // you are 8byte aligned but also 16byte align, therefore you would
514 // either go for the 16byte copy loop OR the ld8 in the tail part.
515 // The combination ld1, ld2, ld4, ld8 where you fail on ld8 is impossible
516 // because it would mean you had 15bytes to copy in which case you
517 // would have defaulted to the byte by byte copy.
518 //
519 //
520 // 2) TAIL
521 // Here we now we have less than 16 bytes AND we are either 8 or 16 byte
522 // aligned.
523 //
524 // Key point:
525 // This means that we either:
526 // - are right on a page boundary
527 // OR
528 // - are at more than 16 bytes from a page boundary with
529 // at most 15 bytes to copy: no chance of crossing.
530 //
531 // This allows us to assume that if we fail on a load we haven't possibly
532 // executed any of the previous (tail) ones, so we don't need to do
533 // any stores. For instance, if we fail on ld2, this means we had
534 // 2 or 3 bytes left to copy and we did not execute the ld8 nor ld4.
535 //
536 // This means that we are in a situation similar the a fault in the
537 // head part. That's nice!
538 //
539 .failure_in1:
540 sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied
541 sub len=endsrc,src1,1
542 //
543 // we know that ret0 can never be zero at this point
544 // because we failed why trying to do a load, i.e. there is still
545 // some work to do.
546 // The failure_in1bis and length problem is taken care of at the
547 // calling side.
548 //
549 ;;
550 .failure_in1bis: // from (.failure_in3)
551 mov ar.lc=len // Continue with a stupid byte store.
552 ;;
553 5:
554 st1 [dst1]=r0,1
555 br.cloop.dptk 5b
556 ;;
557 mov pr=saved_pr,0xffffffffffff0000
558 mov ar.lc=saved_lc
559 mov ar.pfs=saved_pfs
560 br.ret.sptk.many rp
562 //
563 // Here we simply restart the loop but instead
564 // of doing loads we fill the pipeline with zeroes
565 // We can't simply store r0 because we may have valid
566 // data in transit in the pipeline.
567 // ar.lc and ar.ec are setup correctly at this point
568 //
569 // we MUST use src1/endsrc here and not dst1/enddst because
570 // of the pipeline effect.
571 //
572 .failure_in3:
573 sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied
574 ;;
575 2:
576 (p16) mov val1[0]=r0
577 (p16) mov val2[0]=r0
578 (EPI) st8 [dst1]=val1[PIPE_DEPTH-1],16
579 (EPI) st8 [dst2]=val2[PIPE_DEPTH-1],16
580 br.ctop.dptk 2b
581 ;;
582 cmp.ne p6,p0=dst1,enddst // Do we need to finish the tail ?
583 sub len=enddst,dst1,1 // precompute len
584 (p6) br.cond.dptk .failure_in1bis
585 ;;
586 mov pr=saved_pr,0xffffffffffff0000
587 mov ar.lc=saved_lc
588 mov ar.pfs=saved_pfs
589 br.ret.sptk.many rp
591 .failure_in2:
592 sub ret0=endsrc,src1
593 cmp.ne p6,p0=dst1,enddst // Do we need to finish the tail ?
594 sub len=enddst,dst1,1 // precompute len
595 (p6) br.cond.dptk .failure_in1bis
596 ;;
597 mov pr=saved_pr,0xffffffffffff0000
598 mov ar.lc=saved_lc
599 mov ar.pfs=saved_pfs
600 br.ret.sptk.many rp
602 //
603 // handling of failures on stores: that's the easy part
604 //
605 .failure_out:
606 sub ret0=enddst,dst1
607 mov pr=saved_pr,0xffffffffffff0000
608 mov ar.lc=saved_lc
610 mov ar.pfs=saved_pfs
611 br.ret.sptk.many rp
612 END(__copy_user)
613 EXPORT_SYMBOL(__copy_user)