| blob | 76da205282eec7aa1115568beb1f16f184e4a08d |
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * arch/alpha/lib/ev6-stxncpy.S
4 * 21264 version contributed by Rick Gorton <rick.gorton@api-networks.com>
5 *
6 * Copy no more than COUNT bytes of the null-terminated string from
7 * SRC to DST.
8 *
9 * This is an internal routine used by strncpy, stpncpy, and strncat.
10 * As such, it uses special linkage conventions to make implementation
11 * of these public functions more efficient.
12 *
13 * On input:
14 * t9 = return address
15 * a0 = DST
16 * a1 = SRC
17 * a2 = COUNT
18 *
19 * Furthermore, COUNT may not be zero.
20 *
21 * On output:
22 * t0 = last word written
23 * t10 = bitmask (with one bit set) indicating the byte position of
24 * the end of the range specified by COUNT
25 * t12 = bitmask (with one bit set) indicating the last byte written
26 * a0 = unaligned address of the last *word* written
27 * a2 = the number of full words left in COUNT
28 *
29 * Furthermore, v0, a3-a5, t11, and $at are untouched.
30 *
31 * Much of the information about 21264 scheduling/coding comes from:
32 * Compiler Writer's Guide for the Alpha 21264
33 * abbreviated as 'CWG' in other comments here
34 * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
35 * Scheduling notation:
36 * E - either cluster
37 * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1
38 * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1
39 * Try not to change the actual algorithm if possible for consistency.
40 */
42 #include <asm/regdef.h>
44 .set noat
45 .set noreorder
47 .text
49 /* There is a problem with either gdb (as of 4.16) or gas (as of 2.7) that
50 doesn't like putting the entry point for a procedure somewhere in the
51 middle of the procedure descriptor. Work around this by putting the
52 aligned copy in its own procedure descriptor */
55 .ent stxncpy_aligned
56 .align 4
57 stxncpy_aligned:
58 .frame sp, 0, t9, 0
59 .prologue 0
61 /* On entry to this basic block:
62 t0 == the first destination word for masking back in
63 t1 == the first source word. */
65 /* Create the 1st output word and detect 0's in the 1st input word. */
66 lda t2, -1 # E : build a mask against false zero
67 mskqh t2, a1, t2 # U : detection in the src word (stall)
68 mskqh t1, a1, t3 # U :
69 ornot t1, t2, t2 # E : (stall)
71 mskql t0, a1, t0 # U : assemble the first output word
72 cmpbge zero, t2, t8 # E : bits set iff null found
73 or t0, t3, t0 # E : (stall)
74 beq a2, $a_eoc # U :
76 bne t8, $a_eos # U :
77 nop
78 nop
79 nop
81 /* On entry to this basic block:
82 t0 == a source word not containing a null. */
84 /*
85 * nops here to:
86 * separate store quads from load quads
87 * limit of 1 bcond/quad to permit training
88 */
89 $a_loop:
90 stq_u t0, 0(a0) # L :
91 addq a0, 8, a0 # E :
92 subq a2, 1, a2 # E :
93 nop
95 ldq_u t0, 0(a1) # L :
96 addq a1, 8, a1 # E :
97 cmpbge zero, t0, t8 # E :
98 beq a2, $a_eoc # U :
100 beq t8, $a_loop # U :
101 nop
102 nop
103 nop
105 /* Take care of the final (partial) word store. At this point
106 the end-of-count bit is set in t8 iff it applies.
108 On entry to this basic block we have:
109 t0 == the source word containing the null
110 t8 == the cmpbge mask that found it. */
112 $a_eos:
113 negq t8, t12 # E : find low bit set
114 and t8, t12, t12 # E : (stall)
115 /* For the sake of the cache, don't read a destination word
116 if we're not going to need it. */
117 and t12, 0x80, t6 # E : (stall)
118 bne t6, 1f # U : (stall)
120 /* We're doing a partial word store and so need to combine
121 our source and original destination words. */
122 ldq_u t1, 0(a0) # L :
123 subq t12, 1, t6 # E :
124 or t12, t6, t8 # E : (stall)
125 zapnot t0, t8, t0 # U : clear src bytes > null (stall)
127 zap t1, t8, t1 # .. e1 : clear dst bytes <= null
128 or t0, t1, t0 # e1 : (stall)
129 nop
130 nop
132 1: stq_u t0, 0(a0) # L :
133 ret (t9) # L0 : Latency=3
134 nop
135 nop
137 /* Add the end-of-count bit to the eos detection bitmask. */
138 $a_eoc:
139 or t10, t8, t8 # E :
140 br $a_eos # L0 : Latency=3
141 nop
142 nop
144 .end stxncpy_aligned
146 .align 4
147 .ent __stxncpy
148 .globl __stxncpy
149 __stxncpy:
150 .frame sp, 0, t9, 0
151 .prologue 0
153 /* Are source and destination co-aligned? */
154 xor a0, a1, t1 # E :
155 and a0, 7, t0 # E : find dest misalignment
156 and t1, 7, t1 # E : (stall)
157 addq a2, t0, a2 # E : bias count by dest misalignment (stall)
159 subq a2, 1, a2 # E :
160 and a2, 7, t2 # E : (stall)
161 srl a2, 3, a2 # U : a2 = loop counter = (count - 1)/8 (stall)
162 addq zero, 1, t10 # E :
164 sll t10, t2, t10 # U : t10 = bitmask of last count byte
165 bne t1, $unaligned # U :
166 /* We are co-aligned; take care of a partial first word. */
167 ldq_u t1, 0(a1) # L : load first src word
168 addq a1, 8, a1 # E :
170 beq t0, stxncpy_aligned # U : avoid loading dest word if not needed
171 ldq_u t0, 0(a0) # L :
172 nop
173 nop
175 br stxncpy_aligned # .. e1 :
176 nop
177 nop
178 nop
182 /* The source and destination are not co-aligned. Align the destination
183 and cope. We have to be very careful about not reading too much and
184 causing a SEGV. */
186 .align 4
187 $u_head:
188 /* We know just enough now to be able to assemble the first
189 full source word. We can still find a zero at the end of it
190 that prevents us from outputting the whole thing.
192 On entry to this basic block:
193 t0 == the first dest word, unmasked
194 t1 == the shifted low bits of the first source word
195 t6 == bytemask that is -1 in dest word bytes */
197 ldq_u t2, 8(a1) # L : Latency=3 load second src word
198 addq a1, 8, a1 # E :
199 mskql t0, a0, t0 # U : mask trailing garbage in dst
200 extqh t2, a1, t4 # U : (3 cycle stall on t2)
202 or t1, t4, t1 # E : first aligned src word complete (stall)
203 mskqh t1, a0, t1 # U : mask leading garbage in src (stall)
204 or t0, t1, t0 # E : first output word complete (stall)
205 or t0, t6, t6 # E : mask original data for zero test (stall)
207 cmpbge zero, t6, t8 # E :
208 beq a2, $u_eocfin # U :
209 lda t6, -1 # E :
210 nop
212 bne t8, $u_final # U :
213 mskql t6, a1, t6 # U : mask out bits already seen
214 stq_u t0, 0(a0) # L : store first output word
215 or t6, t2, t2 # E : (stall)
217 cmpbge zero, t2, t8 # E : find nulls in second partial
218 addq a0, 8, a0 # E :
219 subq a2, 1, a2 # E :
220 bne t8, $u_late_head_exit # U :
222 /* Finally, we've got all the stupid leading edge cases taken care
223 of and we can set up to enter the main loop. */
224 extql t2, a1, t1 # U : position hi-bits of lo word
225 beq a2, $u_eoc # U :
226 ldq_u t2, 8(a1) # L : read next high-order source word
227 addq a1, 8, a1 # E :
229 extqh t2, a1, t0 # U : position lo-bits of hi word (stall)
230 cmpbge zero, t2, t8 # E :
231 nop
232 bne t8, $u_eos # U :
234 /* Unaligned copy main loop. In order to avoid reading too much,
235 the loop is structured to detect zeros in aligned source words.
236 This has, unfortunately, effectively pulled half of a loop
237 iteration out into the head and half into the tail, but it does
238 prevent nastiness from accumulating in the very thing we want
239 to run as fast as possible.
241 On entry to this basic block:
242 t0 == the shifted low-order bits from the current source word
243 t1 == the shifted high-order bits from the previous source word
244 t2 == the unshifted current source word
246 We further know that t2 does not contain a null terminator. */
248 .align 4
249 $u_loop:
250 or t0, t1, t0 # E : current dst word now complete
251 subq a2, 1, a2 # E : decrement word count
252 extql t2, a1, t1 # U : extract low bits for next time
253 addq a0, 8, a0 # E :
255 stq_u t0, -8(a0) # U : save the current word
256 beq a2, $u_eoc # U :
257 ldq_u t2, 8(a1) # U : Latency=3 load high word for next time
258 addq a1, 8, a1 # E :
260 extqh t2, a1, t0 # U : extract low bits (2 cycle stall)
261 cmpbge zero, t2, t8 # E : test new word for eos
262 nop
263 beq t8, $u_loop # U :
265 /* We've found a zero somewhere in the source word we just read.
266 If it resides in the lower half, we have one (probably partial)
267 word to write out, and if it resides in the upper half, we
268 have one full and one partial word left to write out.
270 On entry to this basic block:
271 t0 == the shifted low-order bits from the current source word
272 t1 == the shifted high-order bits from the previous source word
273 t2 == the unshifted current source word. */
274 $u_eos:
275 or t0, t1, t0 # E : first (partial) source word complete
276 nop
277 cmpbge zero, t0, t8 # E : is the null in this first bit? (stall)
278 bne t8, $u_final # U : (stall)
280 stq_u t0, 0(a0) # L : the null was in the high-order bits
281 addq a0, 8, a0 # E :
282 subq a2, 1, a2 # E :
283 nop
285 $u_late_head_exit:
286 extql t2, a1, t0 # U :
287 cmpbge zero, t0, t8 # E :
288 or t8, t10, t6 # E : (stall)
289 cmoveq a2, t6, t8 # E : Latency=2, extra map slot (stall)
291 /* Take care of a final (probably partial) result word.
292 On entry to this basic block:
293 t0 == assembled source word
294 t8 == cmpbge mask that found the null. */
295 $u_final:
296 negq t8, t6 # E : isolate low bit set
297 and t6, t8, t12 # E : (stall)
298 and t12, 0x80, t6 # E : avoid dest word load if we can (stall)
299 bne t6, 1f # U : (stall)
301 ldq_u t1, 0(a0) # L :
302 subq t12, 1, t6 # E :
303 or t6, t12, t8 # E : (stall)
304 zapnot t0, t8, t0 # U : kill source bytes > null
306 zap t1, t8, t1 # U : kill dest bytes <= null
307 or t0, t1, t0 # E : (stall)
308 nop
309 nop
311 1: stq_u t0, 0(a0) # L :
312 ret (t9) # L0 : Latency=3
314 /* Got to end-of-count before end of string.
315 On entry to this basic block:
316 t1 == the shifted high-order bits from the previous source word */
317 $u_eoc:
318 and a1, 7, t6 # E : avoid final load if possible
319 sll t10, t6, t6 # U : (stall)
320 and t6, 0xff, t6 # E : (stall)
321 bne t6, 1f # U : (stall)
323 ldq_u t2, 8(a1) # L : load final src word
324 nop
325 extqh t2, a1, t0 # U : extract low bits for last word (stall)
326 or t1, t0, t1 # E : (stall)
328 1: cmpbge zero, t1, t8 # E :
329 mov t1, t0 # E :
331 $u_eocfin: # end-of-count, final word
332 or t10, t8, t8 # E :
333 br $u_final # L0 : Latency=3
335 /* Unaligned copy entry point. */
336 .align 4
337 $unaligned:
339 ldq_u t1, 0(a1) # L : load first source word
340 and a0, 7, t4 # E : find dest misalignment
341 and a1, 7, t5 # E : find src misalignment
342 /* Conditionally load the first destination word and a bytemask
343 with 0xff indicating that the destination byte is sacrosanct. */
344 mov zero, t0 # E :
346 mov zero, t6 # E :
347 beq t4, 1f # U :
348 ldq_u t0, 0(a0) # L :
349 lda t6, -1 # E :
351 mskql t6, a0, t6 # U :
352 nop
353 nop
354 subq a1, t4, a1 # E : sub dest misalignment from src addr
356 /* If source misalignment is larger than dest misalignment, we need
357 extra startup checks to avoid SEGV. */
359 1: cmplt t4, t5, t12 # E :
360 extql t1, a1, t1 # U : shift src into place
361 lda t2, -1 # E : for creating masks later
362 beq t12, $u_head # U : (stall)
364 extql t2, a1, t2 # U :
365 cmpbge zero, t1, t8 # E : is there a zero?
366 andnot t2, t6, t2 # E : dest mask for a single word copy
367 or t8, t10, t5 # E : test for end-of-count too
369 cmpbge zero, t2, t3 # E :
370 cmoveq a2, t5, t8 # E : Latency=2, extra map slot
371 nop # E : keep with cmoveq
372 andnot t8, t3, t8 # E : (stall)
374 beq t8, $u_head # U :
375 /* At this point we've found a zero in the first partial word of
376 the source. We need to isolate the valid source data and mask
377 it into the original destination data. (Incidentally, we know
378 that we'll need at least one byte of that original dest word.) */
379 ldq_u t0, 0(a0) # L :
380 negq t8, t6 # E : build bitmask of bytes <= zero
381 mskqh t1, t4, t1 # U :
383 and t6, t8, t12 # E :
384 subq t12, 1, t6 # E : (stall)
385 or t6, t12, t8 # E : (stall)
386 zapnot t2, t8, t2 # U : prepare source word; mirror changes (stall)
388 zapnot t1, t8, t1 # U : to source validity mask
389 andnot t0, t2, t0 # E : zero place for source to reside
390 or t0, t1, t0 # E : and put it there (stall both t0, t1)
391 stq_u t0, 0(a0) # L : (stall)
393 ret (t9) # L0 : Latency=3
394 nop
395 nop
396 nop
398 .end __stxncpy