| blob | 07d10ba428b4d82490a8c59f1c5ec1b1178e74bb |
1 /* checksum.S: Sparc V9 optimized checksum code.
2 *
3 * Copyright(C) 1995 Linus Torvalds
4 * Copyright(C) 1995 Miguel de Icaza
5 * Copyright(C) 1996 David S. Miller
6 * Copyright(C) 1997 Jakub Jelinek
7 *
8 * derived from:
9 * Linux/Alpha checksum c-code
10 * Linux/ix86 inline checksum assembly
11 * RFC1071 Computing the Internet Checksum (esp. Jacobsons m68k code)
12 * David Mosberger-Tang for optimized reference c-code
13 * BSD4.4 portable checksum routine
14 */
16 #include <asm/errno.h>
17 #include <asm/head.h>
18 #include <asm/ptrace.h>
19 #include <asm/asi.h>
20 #include <asm/page.h>
21 #include <asm/asm_offsets.h>
23 /* The problem with the "add with carry" instructions on Ultra
24 * are two fold. Firstly, they cannot pair with jack shit,
25 * and also they only add in the 32-bit carry condition bit
26 * into the accumulated sum. The following is much better.
27 * For larger chunks we use VIS code, which is faster ;)
28 */
30 #define src o0
31 #define dst o1
32 #define len o2
33 #define sum o3
35 .text
36 /* I think I have an erection... Once _AGAIN_ the SunSoft
37 * engineers are caught asleep at the keyboard, tsk tsk...
38 */
40 #define CSUMCOPY_LASTCHUNK(off, t0, t1) \
41 ldxa [%src - off - 0x08] %asi, t0; \
42 ldxa [%src - off - 0x00] %asi, t1; \
43 nop; nop; \
44 addcc t0, %sum, %sum; \
45 stw t0, [%dst - off - 0x04]; \
46 srlx t0, 32, t0; \
47 bcc,pt %xcc, 51f; \
48 stw t0, [%dst - off - 0x08]; \
49 add %sum, 1, %sum; \
50 51: addcc t1, %sum, %sum; \
51 stw t1, [%dst - off + 0x04]; \
52 srlx t1, 32, t1; \
53 bcc,pt %xcc, 52f; \
54 stw t1, [%dst - off - 0x00]; \
55 add %sum, 1, %sum; \
56 52:
58 cpc_start:
59 cc_end_cruft:
60 andcc %g7, 8, %g0 ! IEU1 Group
61 be,pn %icc, 1f ! CTI
62 and %g7, 4, %g5 ! IEU0
63 ldxa [%src + 0x00] %asi, %g2 ! Load Group
64 add %dst, 8, %dst ! IEU0
65 add %src, 8, %src ! IEU1
66 addcc %g2, %sum, %sum ! IEU1 Group + 2 bubbles
67 stw %g2, [%dst - 0x04] ! Store
68 srlx %g2, 32, %g2 ! IEU0
69 bcc,pt %xcc, 1f ! CTI Group
70 stw %g2, [%dst - 0x08] ! Store
71 add %sum, 1, %sum ! IEU0
72 1: brz,pt %g5, 1f ! CTI Group
73 clr %g2 ! IEU0
74 lduwa [%src + 0x00] %asi, %g2 ! Load
75 add %dst, 4, %dst ! IEU0 Group
76 add %src, 4, %src ! IEU1
77 stw %g2, [%dst - 0x04] ! Store Group + 2 bubbles
78 sllx %g2, 32, %g2 ! IEU0
79 1: andcc %g7, 2, %g0 ! IEU1
80 be,pn %icc, 1f ! CTI Group
81 clr %o4 ! IEU1
82 lduha [%src + 0x00] %asi, %o4 ! Load
83 add %src, 2, %src ! IEU0 Group
84 add %dst, 2, %dst ! IEU1
85 sth %o4, [%dst - 0x2] ! Store Group + 2 bubbles
86 sll %o4, 16, %o4 ! IEU0
87 1: andcc %g7, 1, %g0 ! IEU1
88 be,pn %icc, 1f ! CTI Group
89 clr %o5 ! IEU0
90 lduba [%src + 0x00] %asi, %o5 ! Load
91 stb %o5, [%dst + 0x00] ! Store Group + 2 bubbles
92 sll %o5, 8, %o5 ! IEU0
93 1: or %g2, %o4, %o4 ! IEU1
94 or %o5, %o4, %o4 ! IEU0 Group
95 addcc %o4, %sum, %sum ! IEU1
96 bcc,pt %xcc, ccfold ! CTI
97 sethi %uhi(PAGE_OFFSET), %g4 ! IEU0 Group
98 b,pt %xcc, ccfold ! CTI
99 add %sum, 1, %sum ! IEU1
101 cc_fixit:
102 cmp %len, 6 ! IEU1 Group
103 bl,a,pn %icc, ccte ! CTI
104 andcc %len, 0xf, %g7 ! IEU1 Group
105 andcc %src, 2, %g0 ! IEU1 Group
106 be,pn %icc, 1f ! CTI
107 andcc %src, 0x4, %g0 ! IEU1 Group
108 lduha [%src + 0x00] %asi, %g4 ! Load
109 sub %len, 2, %len ! IEU0
110 add %src, 2, %src ! IEU0 Group
111 add %dst, 2, %dst ! IEU1
112 sll %g4, 16, %g3 ! IEU0 Group + 1 bubble
113 addcc %g3, %sum, %sum ! IEU1
114 bcc,pt %xcc, 0f ! CTI
115 srl %sum, 16, %g3 ! IEU0 Group
116 add %g3, 1, %g3 ! IEU0 4 clocks (mispredict)
117 0: andcc %src, 0x4, %g0 ! IEU1 Group
118 sth %g4, [%dst - 0x2] ! Store
119 sll %sum, 16, %sum ! IEU0
120 sll %g3, 16, %g3 ! IEU0 Group
121 srl %sum, 16, %sum ! IEU0 Group
122 or %g3, %sum, %sum ! IEU0 Group (regdep)
123 1: be,pt %icc, ccmerge ! CTI
124 andcc %len, 0xf0, %g1 ! IEU1
125 lduwa [%src + 0x00] %asi, %g4 ! Load Group
126 sub %len, 4, %len ! IEU0
127 add %src, 4, %src ! IEU1
128 add %dst, 4, %dst ! IEU0 Group
129 addcc %g4, %sum, %sum ! IEU1 Group + 1 bubble
130 stw %g4, [%dst - 0x4] ! Store
131 bcc,pt %xcc, ccmerge ! CTI
132 andcc %len, 0xf0, %g1 ! IEU1 Group
133 b,pt %xcc, ccmerge ! CTI 4 clocks (mispredict)
134 add %sum, 1, %sum ! IEU0
136 .align 32
137 .globl csum_partial_copy_sparc64
138 csum_partial_copy_sparc64: /* %o0=src, %o1=dest, %o2=len, %o3=sum */
139 xorcc %src, %dst, %o4 ! IEU1 Group
140 srl %sum, 0, %sum ! IEU0
141 andcc %o4, 3, %g0 ! IEU1 Group
142 srl %len, 0, %len ! IEU0
143 bne,pn %icc, ccslow ! CTI
144 andcc %src, 1, %g0 ! IEU1 Group
145 bne,pn %icc, ccslow ! CTI
146 cmp %len, 256 ! IEU1 Group
147 bgeu,pt %icc, csum_partial_copy_vis ! CTI
148 andcc %src, 7, %g0 ! IEU1 Group
149 bne,pn %icc, cc_fixit ! CTI
150 andcc %len, 0xf0, %g1 ! IEU1 Group
151 ccmerge:be,pn %icc, ccte ! CTI
152 andcc %len, 0xf, %g7 ! IEU1 Group
153 sll %g1, 2, %o4 ! IEU0
154 13: sethi %hi(12f), %o5 ! IEU0 Group
155 add %src, %g1, %src ! IEU1
156 sub %o5, %o4, %o5 ! IEU0 Group
157 jmpl %o5 + %lo(12f), %g0 ! CTI Group brk forced
158 add %dst, %g1, %dst ! IEU0 Group
159 cctbl: CSUMCOPY_LASTCHUNK(0xe8,%g2,%g3)
160 CSUMCOPY_LASTCHUNK(0xd8,%g2,%g3)
161 CSUMCOPY_LASTCHUNK(0xc8,%g2,%g3)
162 CSUMCOPY_LASTCHUNK(0xb8,%g2,%g3)
163 CSUMCOPY_LASTCHUNK(0xa8,%g2,%g3)
164 CSUMCOPY_LASTCHUNK(0x98,%g2,%g3)
165 CSUMCOPY_LASTCHUNK(0x88,%g2,%g3)
166 CSUMCOPY_LASTCHUNK(0x78,%g2,%g3)
167 CSUMCOPY_LASTCHUNK(0x68,%g2,%g3)
168 CSUMCOPY_LASTCHUNK(0x58,%g2,%g3)
169 CSUMCOPY_LASTCHUNK(0x48,%g2,%g3)
170 CSUMCOPY_LASTCHUNK(0x38,%g2,%g3)
171 CSUMCOPY_LASTCHUNK(0x28,%g2,%g3)
172 CSUMCOPY_LASTCHUNK(0x18,%g2,%g3)
173 CSUMCOPY_LASTCHUNK(0x08,%g2,%g3)
174 12:
175 andcc %len, 0xf, %g7 ! IEU1 Group
176 ccte: bne,pn %icc, cc_end_cruft ! CTI
177 sethi %uhi(PAGE_OFFSET), %g4 ! IEU0
178 ccfold: sllx %sum, 32, %o0 ! IEU0 Group
179 addcc %sum, %o0, %o0 ! IEU1 Group (regdep)
180 srlx %o0, 32, %o0 ! IEU0 Group (regdep)
181 bcs,a,pn %xcc, 1f ! CTI
182 add %o0, 1, %o0 ! IEU1 4 clocks (mispredict)
183 1: retl ! CTI Group brk forced
184 sllx %g4, 32, %g4 ! IEU0 Group
186 ccslow: mov 0, %g5
187 brlez,pn %len, 4f
188 andcc %src, 1, %o5
189 be,a,pt %icc, 1f
190 srl %len, 1, %g7
191 sub %len, 1, %len
192 lduba [%src] %asi, %g5
193 add %src, 1, %src
194 stb %g5, [%dst]
195 srl %len, 1, %g7
196 add %dst, 1, %dst
197 1: brz,a,pn %g7, 3f
198 andcc %len, 1, %g0
199 andcc %src, 2, %g0
200 be,a,pt %icc, 1f
201 srl %g7, 1, %g7
202 lduha [%src] %asi, %o4
203 sub %len, 2, %len
204 srl %o4, 8, %g2
205 sub %g7, 1, %g7
206 stb %g2, [%dst]
207 add %o4, %g5, %g5
208 stb %o4, [%dst + 1]
209 add %src, 2, %src
210 srl %g7, 1, %g7
211 add %dst, 2, %dst
212 1: brz,a,pn %g7, 2f
213 andcc %len, 2, %g0
214 lduwa [%src] %asi, %o4
215 5: srl %o4, 24, %g2
216 srl %o4, 16, %g3
217 stb %g2, [%dst]
218 srl %o4, 8, %g2
219 stb %g3, [%dst + 1]
220 add %src, 4, %src
221 stb %g2, [%dst + 2]
222 addcc %o4, %g5, %g5
223 stb %o4, [%dst + 3]
224 addc %g5, %g0, %g5
225 add %dst, 4, %dst
226 subcc %g7, 1, %g7
227 bne,a,pt %icc, 5b
228 lduwa [%src] %asi, %o4
229 sll %g5, 16, %g2
230 srl %g5, 16, %g5
231 srl %g2, 16, %g2
232 andcc %len, 2, %g0
233 add %g2, %g5, %g5
234 2: be,a,pt %icc, 3f
235 andcc %len, 1, %g0
236 lduha [%src] %asi, %o4
237 andcc %len, 1, %g0
238 srl %o4, 8, %g2
239 add %src, 2, %src
240 stb %g2, [%dst]
241 add %g5, %o4, %g5
242 stb %o4, [%dst + 1]
243 add %dst, 2, %dst
244 3: be,a,pt %icc, 1f
245 sll %g5, 16, %o4
246 lduba [%src] %asi, %g2
247 sll %g2, 8, %o4
248 stb %g2, [%dst]
249 add %g5, %o4, %g5
250 sll %g5, 16, %o4
251 1: addcc %o4, %g5, %g5
252 srl %g5, 16, %o4
253 addc %g0, %o4, %g5
254 brz,pt %o5, 4f
255 srl %g5, 8, %o4
256 and %g5, 0xff, %g2
257 and %o4, 0xff, %o4
258 sll %g2, 8, %g2
259 or %g2, %o4, %g5
260 4: addcc %sum, %g5, %sum
261 addc %g0, %sum, %o0
262 retl
263 srl %o0, 0, %o0
264 cpc_end:
266 .globl cpc_handler
267 cpc_handler:
268 ldx [%sp + 0x7ff + 128], %g1
269 ldub [%g6 + AOFF_task_thread + AOFF_thread_current_ds], %g3
270 sub %g0, EFAULT, %g2
271 brnz,a,pt %g1, 1f
272 st %g2, [%g1]
273 1: sethi %uhi(PAGE_OFFSET), %g4
274 wr %g3, %g0, %asi
275 retl
276 sllx %g4, 32, %g4
278 .section __ex_table
279 .align 4
280 .word cpc_start, 0, cpc_end, cpc_handler