| blob | a3c1987f1ecd10bb9c4b72df12aebb6bf0cc0dec |
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 #
3 # Accelerated poly1305 implementation for ppc64le.
4 #
5 # Copyright 2023- IBM Corp. All rights reserved
6 #
7 #===================================================================================
8 # Written by Danny Tsen <dtsen@us.ibm.com>
9 #
10 # Poly1305 - this version mainly using vector/VSX/Scalar
11 # - 26 bits limbs
12 # - Handle multiple 64 byte blcok.
13 #
14 # Block size 16 bytes
15 # key = (r, s)
16 # clamp r &= 0x0FFFFFFC0FFFFFFC 0x0FFFFFFC0FFFFFFF
17 # p = 2^130 - 5
18 # a += m
19 # a = (r + a) % p
20 # a += s
21 #
22 # Improve performance by breaking down polynominal to the sum of products with
23 # h4 = m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r
24 #
25 # 07/22/21 - this revison based on the above sum of products. Setup r^4, r^3, r^2, r and s3, s2, s1, s0
26 # to 9 vectors for multiplications.
27 #
28 # setup r^4, r^3, r^2, r vectors
29 # vs [r^1, r^3, r^2, r^4]
30 # vs0 = [r0,.....]
31 # vs1 = [r1,.....]
32 # vs2 = [r2,.....]
33 # vs3 = [r3,.....]
34 # vs4 = [r4,.....]
35 # vs5 = [r1*5,...]
36 # vs6 = [r2*5,...]
37 # vs7 = [r2*5,...]
38 # vs8 = [r4*5,...]
39 #
40 # Each word in a vector consists a member of a "r/s" in [a * r/s].
41 #
42 # r0, r4*5, r3*5, r2*5, r1*5;
43 # r1, r0, r4*5, r3*5, r2*5;
44 # r2, r1, r0, r4*5, r3*5;
45 # r3, r2, r1, r0, r4*5;
46 # r4, r3, r2, r1, r0 ;
47 #
48 #
49 # poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m)
50 # k = 32 bytes key
51 # r3 = k (r, s)
52 # r4 = mlen
53 # r5 = m
54 #
55 #include <asm/ppc_asm.h>
56 #include <asm/asm-offsets.h>
57 #include <asm/asm-compat.h>
58 #include <linux/linkage.h>
60 .machine "any"
62 .text
64 .macro SAVE_GPR GPR OFFSET FRAME
65 std \GPR,\OFFSET(\FRAME)
66 .endm
68 .macro SAVE_VRS VRS OFFSET FRAME
69 li 16, \OFFSET
70 stvx \VRS, 16, \FRAME
71 .endm
73 .macro SAVE_VSX VSX OFFSET FRAME
74 li 16, \OFFSET
75 stxvx \VSX, 16, \FRAME
76 .endm
78 .macro RESTORE_GPR GPR OFFSET FRAME
79 ld \GPR,\OFFSET(\FRAME)
80 .endm
82 .macro RESTORE_VRS VRS OFFSET FRAME
83 li 16, \OFFSET
84 lvx \VRS, 16, \FRAME
85 .endm
87 .macro RESTORE_VSX VSX OFFSET FRAME
88 li 16, \OFFSET
89 lxvx \VSX, 16, \FRAME
90 .endm
92 .macro SAVE_REGS
93 mflr 0
94 std 0, 16(1)
95 stdu 1,-752(1)
97 SAVE_GPR 14, 112, 1
98 SAVE_GPR 15, 120, 1
99 SAVE_GPR 16, 128, 1
100 SAVE_GPR 17, 136, 1
101 SAVE_GPR 18, 144, 1
102 SAVE_GPR 19, 152, 1
103 SAVE_GPR 20, 160, 1
104 SAVE_GPR 21, 168, 1
105 SAVE_GPR 22, 176, 1
106 SAVE_GPR 23, 184, 1
107 SAVE_GPR 24, 192, 1
108 SAVE_GPR 25, 200, 1
109 SAVE_GPR 26, 208, 1
110 SAVE_GPR 27, 216, 1
111 SAVE_GPR 28, 224, 1
112 SAVE_GPR 29, 232, 1
113 SAVE_GPR 30, 240, 1
114 SAVE_GPR 31, 248, 1
116 addi 9, 1, 256
117 SAVE_VRS 20, 0, 9
118 SAVE_VRS 21, 16, 9
119 SAVE_VRS 22, 32, 9
120 SAVE_VRS 23, 48, 9
121 SAVE_VRS 24, 64, 9
122 SAVE_VRS 25, 80, 9
123 SAVE_VRS 26, 96, 9
124 SAVE_VRS 27, 112, 9
125 SAVE_VRS 28, 128, 9
126 SAVE_VRS 29, 144, 9
127 SAVE_VRS 30, 160, 9
128 SAVE_VRS 31, 176, 9
130 SAVE_VSX 14, 192, 9
131 SAVE_VSX 15, 208, 9
132 SAVE_VSX 16, 224, 9
133 SAVE_VSX 17, 240, 9
134 SAVE_VSX 18, 256, 9
135 SAVE_VSX 19, 272, 9
136 SAVE_VSX 20, 288, 9
137 SAVE_VSX 21, 304, 9
138 SAVE_VSX 22, 320, 9
139 SAVE_VSX 23, 336, 9
140 SAVE_VSX 24, 352, 9
141 SAVE_VSX 25, 368, 9
142 SAVE_VSX 26, 384, 9
143 SAVE_VSX 27, 400, 9
144 SAVE_VSX 28, 416, 9
145 SAVE_VSX 29, 432, 9
146 SAVE_VSX 30, 448, 9
147 SAVE_VSX 31, 464, 9
148 .endm # SAVE_REGS
150 .macro RESTORE_REGS
151 addi 9, 1, 256
152 RESTORE_VRS 20, 0, 9
153 RESTORE_VRS 21, 16, 9
154 RESTORE_VRS 22, 32, 9
155 RESTORE_VRS 23, 48, 9
156 RESTORE_VRS 24, 64, 9
157 RESTORE_VRS 25, 80, 9
158 RESTORE_VRS 26, 96, 9
159 RESTORE_VRS 27, 112, 9
160 RESTORE_VRS 28, 128, 9
161 RESTORE_VRS 29, 144, 9
162 RESTORE_VRS 30, 160, 9
163 RESTORE_VRS 31, 176, 9
165 RESTORE_VSX 14, 192, 9
166 RESTORE_VSX 15, 208, 9
167 RESTORE_VSX 16, 224, 9
168 RESTORE_VSX 17, 240, 9
169 RESTORE_VSX 18, 256, 9
170 RESTORE_VSX 19, 272, 9
171 RESTORE_VSX 20, 288, 9
172 RESTORE_VSX 21, 304, 9
173 RESTORE_VSX 22, 320, 9
174 RESTORE_VSX 23, 336, 9
175 RESTORE_VSX 24, 352, 9
176 RESTORE_VSX 25, 368, 9
177 RESTORE_VSX 26, 384, 9
178 RESTORE_VSX 27, 400, 9
179 RESTORE_VSX 28, 416, 9
180 RESTORE_VSX 29, 432, 9
181 RESTORE_VSX 30, 448, 9
182 RESTORE_VSX 31, 464, 9
184 RESTORE_GPR 14, 112, 1
185 RESTORE_GPR 15, 120, 1
186 RESTORE_GPR 16, 128, 1
187 RESTORE_GPR 17, 136, 1
188 RESTORE_GPR 18, 144, 1
189 RESTORE_GPR 19, 152, 1
190 RESTORE_GPR 20, 160, 1
191 RESTORE_GPR 21, 168, 1
192 RESTORE_GPR 22, 176, 1
193 RESTORE_GPR 23, 184, 1
194 RESTORE_GPR 24, 192, 1
195 RESTORE_GPR 25, 200, 1
196 RESTORE_GPR 26, 208, 1
197 RESTORE_GPR 27, 216, 1
198 RESTORE_GPR 28, 224, 1
199 RESTORE_GPR 29, 232, 1
200 RESTORE_GPR 30, 240, 1
201 RESTORE_GPR 31, 248, 1
203 addi 1, 1, 752
204 ld 0, 16(1)
205 mtlr 0
206 .endm # RESTORE_REGS
208 #
209 # p[0] = a0*r0 + a1*r4*5 + a2*r3*5 + a3*r2*5 + a4*r1*5;
210 # p[1] = a0*r1 + a1*r0 + a2*r4*5 + a3*r3*5 + a4*r2*5;
211 # p[2] = a0*r2 + a1*r1 + a2*r0 + a3*r4*5 + a4*r3*5;
212 # p[3] = a0*r3 + a1*r2 + a2*r1 + a3*r0 + a4*r4*5;
213 # p[4] = a0*r4 + a1*r3 + a2*r2 + a3*r1 + a4*r0 ;
214 #
215 # [r^2, r^3, r^1, r^4]
216 # [m3, m2, m4, m1]
217 #
218 # multiply odd and even words
219 .macro mul_odd
220 vmulouw 14, 4, 26
221 vmulouw 10, 5, 3
222 vmulouw 11, 6, 2
223 vmulouw 12, 7, 1
224 vmulouw 13, 8, 0
225 vmulouw 15, 4, 27
226 vaddudm 14, 14, 10
227 vaddudm 14, 14, 11
228 vmulouw 10, 5, 26
229 vmulouw 11, 6, 3
230 vaddudm 14, 14, 12
231 vaddudm 14, 14, 13 # x0
232 vaddudm 15, 15, 10
233 vaddudm 15, 15, 11
234 vmulouw 12, 7, 2
235 vmulouw 13, 8, 1
236 vaddudm 15, 15, 12
237 vaddudm 15, 15, 13 # x1
238 vmulouw 16, 4, 28
239 vmulouw 10, 5, 27
240 vmulouw 11, 6, 26
241 vaddudm 16, 16, 10
242 vaddudm 16, 16, 11
243 vmulouw 12, 7, 3
244 vmulouw 13, 8, 2
245 vaddudm 16, 16, 12
246 vaddudm 16, 16, 13 # x2
247 vmulouw 17, 4, 29
248 vmulouw 10, 5, 28
249 vmulouw 11, 6, 27
250 vaddudm 17, 17, 10
251 vaddudm 17, 17, 11
252 vmulouw 12, 7, 26
253 vmulouw 13, 8, 3
254 vaddudm 17, 17, 12
255 vaddudm 17, 17, 13 # x3
256 vmulouw 18, 4, 30
257 vmulouw 10, 5, 29
258 vmulouw 11, 6, 28
259 vaddudm 18, 18, 10
260 vaddudm 18, 18, 11
261 vmulouw 12, 7, 27
262 vmulouw 13, 8, 26
263 vaddudm 18, 18, 12
264 vaddudm 18, 18, 13 # x4
265 .endm
267 .macro mul_even
268 vmuleuw 9, 4, 26
269 vmuleuw 10, 5, 3
270 vmuleuw 11, 6, 2
271 vmuleuw 12, 7, 1
272 vmuleuw 13, 8, 0
273 vaddudm 14, 14, 9
274 vaddudm 14, 14, 10
275 vaddudm 14, 14, 11
276 vaddudm 14, 14, 12
277 vaddudm 14, 14, 13 # x0
279 vmuleuw 9, 4, 27
280 vmuleuw 10, 5, 26
281 vmuleuw 11, 6, 3
282 vmuleuw 12, 7, 2
283 vmuleuw 13, 8, 1
284 vaddudm 15, 15, 9
285 vaddudm 15, 15, 10
286 vaddudm 15, 15, 11
287 vaddudm 15, 15, 12
288 vaddudm 15, 15, 13 # x1
290 vmuleuw 9, 4, 28
291 vmuleuw 10, 5, 27
292 vmuleuw 11, 6, 26
293 vmuleuw 12, 7, 3
294 vmuleuw 13, 8, 2
295 vaddudm 16, 16, 9
296 vaddudm 16, 16, 10
297 vaddudm 16, 16, 11
298 vaddudm 16, 16, 12
299 vaddudm 16, 16, 13 # x2
301 vmuleuw 9, 4, 29
302 vmuleuw 10, 5, 28
303 vmuleuw 11, 6, 27
304 vmuleuw 12, 7, 26
305 vmuleuw 13, 8, 3
306 vaddudm 17, 17, 9
307 vaddudm 17, 17, 10
308 vaddudm 17, 17, 11
309 vaddudm 17, 17, 12
310 vaddudm 17, 17, 13 # x3
312 vmuleuw 9, 4, 30
313 vmuleuw 10, 5, 29
314 vmuleuw 11, 6, 28
315 vmuleuw 12, 7, 27
316 vmuleuw 13, 8, 26
317 vaddudm 18, 18, 9
318 vaddudm 18, 18, 10
319 vaddudm 18, 18, 11
320 vaddudm 18, 18, 12
321 vaddudm 18, 18, 13 # x4
322 .endm
324 #
325 # poly1305_setup_r
326 #
327 # setup r^4, r^3, r^2, r vectors
328 # [r, r^3, r^2, r^4]
329 # vs0 = [r0,...]
330 # vs1 = [r1,...]
331 # vs2 = [r2,...]
332 # vs3 = [r3,...]
333 # vs4 = [r4,...]
334 # vs5 = [r4*5,...]
335 # vs6 = [r3*5,...]
336 # vs7 = [r2*5,...]
337 # vs8 = [r1*5,...]
338 #
339 # r0, r4*5, r3*5, r2*5, r1*5;
340 # r1, r0, r4*5, r3*5, r2*5;
341 # r2, r1, r0, r4*5, r3*5;
342 # r3, r2, r1, r0, r4*5;
343 # r4, r3, r2, r1, r0 ;
344 #
345 .macro poly1305_setup_r
347 # save r
348 xxlor 26, 58, 58
349 xxlor 27, 59, 59
350 xxlor 28, 60, 60
351 xxlor 29, 61, 61
352 xxlor 30, 62, 62
354 xxlxor 31, 31, 31
356 # [r, r^3, r^2, r^4]
357 # compute r^2
358 vmr 4, 26
359 vmr 5, 27
360 vmr 6, 28
361 vmr 7, 29
362 vmr 8, 30
363 bl do_mul # r^2 r^1
364 xxpermdi 58, 58, 36, 0x3 # r0
365 xxpermdi 59, 59, 37, 0x3 # r1
366 xxpermdi 60, 60, 38, 0x3 # r2
367 xxpermdi 61, 61, 39, 0x3 # r3
368 xxpermdi 62, 62, 40, 0x3 # r4
369 xxpermdi 36, 36, 36, 0x3
370 xxpermdi 37, 37, 37, 0x3
371 xxpermdi 38, 38, 38, 0x3
372 xxpermdi 39, 39, 39, 0x3
373 xxpermdi 40, 40, 40, 0x3
374 vspltisb 13, 2
375 vsld 9, 27, 13
376 vsld 10, 28, 13
377 vsld 11, 29, 13
378 vsld 12, 30, 13
379 vaddudm 0, 9, 27
380 vaddudm 1, 10, 28
381 vaddudm 2, 11, 29
382 vaddudm 3, 12, 30
384 bl do_mul # r^4 r^3
385 vmrgow 26, 26, 4
386 vmrgow 27, 27, 5
387 vmrgow 28, 28, 6
388 vmrgow 29, 29, 7
389 vmrgow 30, 30, 8
390 vspltisb 13, 2
391 vsld 9, 27, 13
392 vsld 10, 28, 13
393 vsld 11, 29, 13
394 vsld 12, 30, 13
395 vaddudm 0, 9, 27
396 vaddudm 1, 10, 28
397 vaddudm 2, 11, 29
398 vaddudm 3, 12, 30
400 # r^2 r^4
401 xxlor 0, 58, 58
402 xxlor 1, 59, 59
403 xxlor 2, 60, 60
404 xxlor 3, 61, 61
405 xxlor 4, 62, 62
406 xxlor 5, 32, 32
407 xxlor 6, 33, 33
408 xxlor 7, 34, 34
409 xxlor 8, 35, 35
411 vspltw 9, 26, 3
412 vspltw 10, 26, 2
413 vmrgow 26, 10, 9
414 vspltw 9, 27, 3
415 vspltw 10, 27, 2
416 vmrgow 27, 10, 9
417 vspltw 9, 28, 3
418 vspltw 10, 28, 2
419 vmrgow 28, 10, 9
420 vspltw 9, 29, 3
421 vspltw 10, 29, 2
422 vmrgow 29, 10, 9
423 vspltw 9, 30, 3
424 vspltw 10, 30, 2
425 vmrgow 30, 10, 9
427 vsld 9, 27, 13
428 vsld 10, 28, 13
429 vsld 11, 29, 13
430 vsld 12, 30, 13
431 vaddudm 0, 9, 27
432 vaddudm 1, 10, 28
433 vaddudm 2, 11, 29
434 vaddudm 3, 12, 30
435 .endm
437 SYM_FUNC_START_LOCAL(do_mul)
438 mul_odd
440 # do reduction ( h %= p )
441 # carry reduction
442 vspltisb 9, 2
443 vsrd 10, 14, 31
444 vsrd 11, 17, 31
445 vand 7, 17, 25
446 vand 4, 14, 25
447 vaddudm 18, 18, 11
448 vsrd 12, 18, 31
449 vaddudm 15, 15, 10
451 vsrd 11, 15, 31
452 vand 8, 18, 25
453 vand 5, 15, 25
454 vaddudm 4, 4, 12
455 vsld 10, 12, 9
456 vaddudm 6, 16, 11
458 vsrd 13, 6, 31
459 vand 6, 6, 25
460 vaddudm 4, 4, 10
461 vsrd 10, 4, 31
462 vaddudm 7, 7, 13
464 vsrd 11, 7, 31
465 vand 7, 7, 25
466 vand 4, 4, 25
467 vaddudm 5, 5, 10
468 vaddudm 8, 8, 11
469 blr
470 SYM_FUNC_END(do_mul)
472 #
473 # init key
474 #
475 .macro do_poly1305_init
476 addis 10, 2, rmask@toc@ha
477 addi 10, 10, rmask@toc@l
479 ld 11, 0(10)
480 ld 12, 8(10)
482 li 14, 16
483 li 15, 32
484 addis 10, 2, cnum@toc@ha
485 addi 10, 10, cnum@toc@l
486 lvx 25, 0, 10 # v25 - mask
487 lvx 31, 14, 10 # v31 = 1a
488 lvx 19, 15, 10 # v19 = 1 << 24
489 lxv 24, 48(10) # vs24
490 lxv 25, 64(10) # vs25
492 # initialize
493 # load key from r3 to vectors
494 ld 9, 24(3)
495 ld 10, 32(3)
496 and. 9, 9, 11
497 and. 10, 10, 12
499 # break 26 bits
500 extrdi 14, 9, 26, 38
501 extrdi 15, 9, 26, 12
502 extrdi 16, 9, 12, 0
503 mtvsrdd 58, 0, 14
504 insrdi 16, 10, 14, 38
505 mtvsrdd 59, 0, 15
506 extrdi 17, 10, 26, 24
507 mtvsrdd 60, 0, 16
508 extrdi 18, 10, 24, 0
509 mtvsrdd 61, 0, 17
510 mtvsrdd 62, 0, 18
512 # r1 = r1 * 5, r2 = r2 * 5, r3 = r3 * 5, r4 = r4 * 5
513 li 9, 5
514 mtvsrdd 36, 0, 9
515 vmulouw 0, 27, 4 # v0 = rr0
516 vmulouw 1, 28, 4 # v1 = rr1
517 vmulouw 2, 29, 4 # v2 = rr2
518 vmulouw 3, 30, 4 # v3 = rr3
519 .endm
521 #
522 # poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m)
523 # k = 32 bytes key
524 # r3 = k (r, s)
525 # r4 = mlen
526 # r5 = m
527 #
528 SYM_FUNC_START(poly1305_p10le_4blocks)
529 .align 5
530 cmpdi 5, 64
531 blt Out_no_poly1305
533 SAVE_REGS
535 do_poly1305_init
537 li 21, 0 # counter to message
539 poly1305_setup_r
541 # load previous H state
542 # break/convert r6 to 26 bits
543 ld 9, 0(3)
544 ld 10, 8(3)
545 ld 19, 16(3)
546 sldi 19, 19, 24
547 mtvsrdd 41, 0, 19
548 extrdi 14, 9, 26, 38
549 extrdi 15, 9, 26, 12
550 extrdi 16, 9, 12, 0
551 mtvsrdd 36, 0, 14
552 insrdi 16, 10, 14, 38
553 mtvsrdd 37, 0, 15
554 extrdi 17, 10, 26, 24
555 mtvsrdd 38, 0, 16
556 extrdi 18, 10, 24, 0
557 mtvsrdd 39, 0, 17
558 mtvsrdd 40, 0, 18
559 vor 8, 8, 9
561 # input m1 m2
562 add 20, 4, 21
563 xxlor 49, 24, 24
564 xxlor 50, 25, 25
565 lxvw4x 43, 0, 20
566 addi 17, 20, 16
567 lxvw4x 44, 0, 17
568 vperm 14, 11, 12, 17
569 vperm 15, 11, 12, 18
570 vand 9, 14, 25 # a0
571 vsrd 10, 14, 31 # >> 26
572 vsrd 11, 10, 31 # 12 bits left
573 vand 10, 10, 25 # a1
574 vspltisb 13, 12
575 vand 16, 15, 25
576 vsld 12, 16, 13
577 vor 11, 11, 12
578 vand 11, 11, 25 # a2
579 vspltisb 13, 14
580 vsrd 12, 15, 13 # >> 14
581 vsrd 13, 12, 31 # >> 26, a4
582 vand 12, 12, 25 # a3
584 vaddudm 20, 4, 9
585 vaddudm 21, 5, 10
586 vaddudm 22, 6, 11
587 vaddudm 23, 7, 12
588 vaddudm 24, 8, 13
590 # m3 m4
591 addi 17, 17, 16
592 lxvw4x 43, 0, 17
593 addi 17, 17, 16
594 lxvw4x 44, 0, 17
595 vperm 14, 11, 12, 17
596 vperm 15, 11, 12, 18
597 vand 9, 14, 25 # a0
598 vsrd 10, 14, 31 # >> 26
599 vsrd 11, 10, 31 # 12 bits left
600 vand 10, 10, 25 # a1
601 vspltisb 13, 12
602 vand 16, 15, 25
603 vsld 12, 16, 13
604 vspltisb 13, 14
605 vor 11, 11, 12
606 vand 11, 11, 25 # a2
607 vsrd 12, 15, 13 # >> 14
608 vsrd 13, 12, 31 # >> 26, a4
609 vand 12, 12, 25 # a3
611 # Smash 4 message blocks into 5 vectors of [m4, m2, m3, m1]
612 vmrgow 4, 9, 20
613 vmrgow 5, 10, 21
614 vmrgow 6, 11, 22
615 vmrgow 7, 12, 23
616 vmrgow 8, 13, 24
617 vaddudm 8, 8, 19
619 addi 5, 5, -64 # len -= 64
620 addi 21, 21, 64 # offset += 64
622 li 9, 64
623 divdu 31, 5, 9
625 cmpdi 31, 0
626 ble Skip_block_loop
628 mtctr 31
630 # h4 = m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r
631 # Rewrite the polynominal sum of product as follows,
632 # h1 = (h0 + m1) * r^2, h2 = (h0 + m2) * r^2
633 # h3 = (h1 + m3) * r^2, h4 = (h2 + m4) * r^2 --> (h0 + m1) r*4 + (h3 + m3) r^2, (h0 + m2) r^4 + (h0 + m4) r^2
634 # .... Repeat
635 # h5 = (h3 + m5) * r^2, h6 = (h4 + m6) * r^2 -->
636 # h7 = (h5 + m7) * r^2, h8 = (h6 + m8) * r^1 --> m5 * r^4 + m6 * r^3 + m7 * r^2 + m8 * r
637 #
638 loop_4blocks:
640 # Multiply odd words and even words
641 mul_odd
642 mul_even
643 # carry reduction
644 vspltisb 9, 2
645 vsrd 10, 14, 31
646 vsrd 11, 17, 31
647 vand 7, 17, 25
648 vand 4, 14, 25
649 vaddudm 18, 18, 11
650 vsrd 12, 18, 31
651 vaddudm 15, 15, 10
653 vsrd 11, 15, 31
654 vand 8, 18, 25
655 vand 5, 15, 25
656 vaddudm 4, 4, 12
657 vsld 10, 12, 9
658 vaddudm 6, 16, 11
660 vsrd 13, 6, 31
661 vand 6, 6, 25
662 vaddudm 4, 4, 10
663 vsrd 10, 4, 31
664 vaddudm 7, 7, 13
666 vsrd 11, 7, 31
667 vand 7, 7, 25
668 vand 4, 4, 25
669 vaddudm 5, 5, 10
670 vaddudm 8, 8, 11
672 # input m1 m2 m3 m4
673 add 20, 4, 21
674 xxlor 49, 24, 24
675 xxlor 50, 25, 25
676 lxvw4x 43, 0, 20
677 addi 17, 20, 16
678 lxvw4x 44, 0, 17
679 vperm 14, 11, 12, 17
680 vperm 15, 11, 12, 18
681 addi 17, 17, 16
682 lxvw4x 43, 0, 17
683 addi 17, 17, 16
684 lxvw4x 44, 0, 17
685 vperm 17, 11, 12, 17
686 vperm 18, 11, 12, 18
688 vand 20, 14, 25 # a0
689 vand 9, 17, 25 # a0
690 vsrd 21, 14, 31 # >> 26
691 vsrd 22, 21, 31 # 12 bits left
692 vsrd 10, 17, 31 # >> 26
693 vsrd 11, 10, 31 # 12 bits left
695 vand 21, 21, 25 # a1
696 vand 10, 10, 25 # a1
698 vspltisb 13, 12
699 vand 16, 15, 25
700 vsld 23, 16, 13
701 vor 22, 22, 23
702 vand 22, 22, 25 # a2
703 vand 16, 18, 25
704 vsld 12, 16, 13
705 vor 11, 11, 12
706 vand 11, 11, 25 # a2
707 vspltisb 13, 14
708 vsrd 23, 15, 13 # >> 14
709 vsrd 24, 23, 31 # >> 26, a4
710 vand 23, 23, 25 # a3
711 vsrd 12, 18, 13 # >> 14
712 vsrd 13, 12, 31 # >> 26, a4
713 vand 12, 12, 25 # a3
715 vaddudm 4, 4, 20
716 vaddudm 5, 5, 21
717 vaddudm 6, 6, 22
718 vaddudm 7, 7, 23
719 vaddudm 8, 8, 24
721 # Smash 4 message blocks into 5 vectors of [m4, m2, m3, m1]
722 vmrgow 4, 9, 4
723 vmrgow 5, 10, 5
724 vmrgow 6, 11, 6
725 vmrgow 7, 12, 7
726 vmrgow 8, 13, 8
727 vaddudm 8, 8, 19
729 addi 5, 5, -64 # len -= 64
730 addi 21, 21, 64 # offset += 64
732 bdnz loop_4blocks
734 Skip_block_loop:
735 xxlor 58, 0, 0
736 xxlor 59, 1, 1
737 xxlor 60, 2, 2
738 xxlor 61, 3, 3
739 xxlor 62, 4, 4
740 xxlor 32, 5, 5
741 xxlor 33, 6, 6
742 xxlor 34, 7, 7
743 xxlor 35, 8, 8
745 # Multiply odd words and even words
746 mul_odd
747 mul_even
749 # Sum the products.
750 xxpermdi 41, 31, 46, 0
751 xxpermdi 42, 31, 47, 0
752 vaddudm 4, 14, 9
753 xxpermdi 36, 31, 36, 3
754 vaddudm 5, 15, 10
755 xxpermdi 37, 31, 37, 3
756 xxpermdi 43, 31, 48, 0
757 vaddudm 6, 16, 11
758 xxpermdi 38, 31, 38, 3
759 xxpermdi 44, 31, 49, 0
760 vaddudm 7, 17, 12
761 xxpermdi 39, 31, 39, 3
762 xxpermdi 45, 31, 50, 0
763 vaddudm 8, 18, 13
764 xxpermdi 40, 31, 40, 3
766 # carry reduction
767 vspltisb 9, 2
768 vsrd 10, 4, 31
769 vsrd 11, 7, 31
770 vand 7, 7, 25
771 vand 4, 4, 25
772 vaddudm 8, 8, 11
773 vsrd 12, 8, 31
774 vaddudm 5, 5, 10
776 vsrd 11, 5, 31
777 vand 8, 8, 25
778 vand 5, 5, 25
779 vaddudm 4, 4, 12
780 vsld 10, 12, 9
781 vaddudm 6, 6, 11
783 vsrd 13, 6, 31
784 vand 6, 6, 25
785 vaddudm 4, 4, 10
786 vsrd 10, 4, 31
787 vaddudm 7, 7, 13
789 vsrd 11, 7, 31
790 vand 7, 7, 25
791 vand 4, 4, 25
792 vaddudm 5, 5, 10
793 vsrd 10, 5, 31
794 vand 5, 5, 25
795 vaddudm 6, 6, 10
796 vaddudm 8, 8, 11
798 b do_final_update
800 do_final_update:
801 # combine 26 bit limbs
802 # v4, v5, v6, v7 and v8 are 26 bit vectors
803 vsld 5, 5, 31
804 vor 20, 4, 5
805 vspltisb 11, 12
806 vsrd 12, 6, 11
807 vsld 6, 6, 31
808 vsld 6, 6, 31
809 vor 20, 20, 6
810 vspltisb 11, 14
811 vsld 7, 7, 11
812 vor 21, 7, 12
813 mfvsrld 16, 40 # save last 2 bytes
814 vsld 8, 8, 11
815 vsld 8, 8, 31
816 vor 21, 21, 8
817 mfvsrld 17, 52
818 mfvsrld 19, 53
819 srdi 16, 16, 24
821 std 17, 0(3)
822 std 19, 8(3)
823 stw 16, 16(3)
825 Out_loop:
826 li 3, 0
828 RESTORE_REGS
830 blr
832 Out_no_poly1305:
833 li 3, 0
834 blr
835 SYM_FUNC_END(poly1305_p10le_4blocks)
837 #
838 # =======================================================================
839 # The following functions implement 64 x 64 bits multiplication poly1305.
840 #
841 SYM_FUNC_START_LOCAL(Poly1305_init_64)
842 # mask 0x0FFFFFFC0FFFFFFC
843 # mask 0x0FFFFFFC0FFFFFFF
844 addis 10, 2, rmask@toc@ha
845 addi 10, 10, rmask@toc@l
846 ld 11, 0(10)
847 ld 12, 8(10)
849 # initialize
850 # load key from r3
851 ld 9, 24(3)
852 ld 10, 32(3)
853 and. 9, 9, 11 # cramp mask r0
854 and. 10, 10, 12 # cramp mask r1
856 srdi 21, 10, 2
857 add 19, 21, 10 # s1: r19 - (r1 >> 2) *5
859 # setup r and s
860 li 25, 0
861 mtvsrdd 32+0, 9, 19 # r0, s1
862 mtvsrdd 32+1, 10, 9 # r1, r0
863 mtvsrdd 32+2, 19, 25 # s1
864 mtvsrdd 32+3, 9, 25 # r0
866 blr
867 SYM_FUNC_END(Poly1305_init_64)
869 # Poly1305_mult
870 # v6 = (h0, h1), v8 = h2
871 # v0 = (r0, s1), v1 = (r1, r0), v2 = s1, v3 = r0
872 #
873 # Output: v7, v10, v11
874 #
875 SYM_FUNC_START_LOCAL(Poly1305_mult)
876 #
877 # d0 = h0 * r0 + h1 * s1
878 vmsumudm 7, 6, 0, 9 # h0 * r0, h1 * s1
880 # d1 = h0 * r1 + h1 * r0 + h2 * s1
881 vmsumudm 11, 6, 1, 9 # h0 * r1, h1 * r0
882 vmsumudm 10, 8, 2, 11 # d1 += h2 * s1
884 # d2 = r0
885 vmsumudm 11, 8, 3, 9 # d2 = h2 * r0
886 blr
887 SYM_FUNC_END(Poly1305_mult)
889 #
890 # carry reduction
891 # h %=p
892 #
893 # Input: v7, v10, v11
894 # Output: r27, r28, r29
895 #
896 SYM_FUNC_START_LOCAL(Carry_reduction)
897 mfvsrld 27, 32+7
898 mfvsrld 28, 32+10
899 mfvsrld 29, 32+11
900 mfvsrd 20, 32+7 # h0.h
901 mfvsrd 21, 32+10 # h1.h
903 addc 28, 28, 20
904 adde 29, 29, 21
905 srdi 22, 29, 0x2
906 sldi 23, 22, 0x2
907 add 23, 23, 22 # (h2 & 3) * 5
908 addc 27, 27, 23 # h0
909 addze 28, 28 # h1
910 andi. 29, 29, 0x3 # h2
911 blr
912 SYM_FUNC_END(Carry_reduction)
914 #
915 # poly1305 multiplication
916 # h *= r, h %= p
917 # d0 = h0 * r0 + h1 * s1
918 # d1 = h0 * r1 + h1 * r0 + h2 * s1
919 # d2 = h0 * r0
920 #
921 #
922 # unsigned int poly1305_test_64s(unisgned char *state, const byte *src, size_t len, highbit)
923 # - no highbit if final leftover block (highbit = 0)
924 #
925 SYM_FUNC_START(poly1305_64s)
926 cmpdi 5, 0
927 ble Out_no_poly1305_64
929 mflr 0
930 std 0, 16(1)
931 stdu 1,-400(1)
933 SAVE_GPR 14, 112, 1
934 SAVE_GPR 15, 120, 1
935 SAVE_GPR 16, 128, 1
936 SAVE_GPR 17, 136, 1
937 SAVE_GPR 18, 144, 1
938 SAVE_GPR 19, 152, 1
939 SAVE_GPR 20, 160, 1
940 SAVE_GPR 21, 168, 1
941 SAVE_GPR 22, 176, 1
942 SAVE_GPR 23, 184, 1
943 SAVE_GPR 24, 192, 1
944 SAVE_GPR 25, 200, 1
945 SAVE_GPR 26, 208, 1
946 SAVE_GPR 27, 216, 1
947 SAVE_GPR 28, 224, 1
948 SAVE_GPR 29, 232, 1
949 SAVE_GPR 30, 240, 1
950 SAVE_GPR 31, 248, 1
952 # Init poly1305
953 bl Poly1305_init_64
955 li 25, 0 # offset to inp and outp
957 add 11, 25, 4
959 # load h
960 # h0, h1, h2?
961 ld 27, 0(3)
962 ld 28, 8(3)
963 lwz 29, 16(3)
965 li 30, 16
966 divdu 31, 5, 30
968 mtctr 31
970 mr 24, 6 # highbit
972 Loop_block_64:
973 vxor 9, 9, 9
975 ld 20, 0(11)
976 ld 21, 8(11)
977 addi 11, 11, 16
979 addc 27, 27, 20
980 adde 28, 28, 21
981 adde 29, 29, 24
983 li 22, 0
984 mtvsrdd 32+6, 27, 28 # h0, h1
985 mtvsrdd 32+8, 29, 22 # h2
987 bl Poly1305_mult
989 bl Carry_reduction
991 bdnz Loop_block_64
993 std 27, 0(3)
994 std 28, 8(3)
995 stw 29, 16(3)
997 li 3, 0
999 RESTORE_GPR 14, 112, 1
1000 RESTORE_GPR 15, 120, 1
1001 RESTORE_GPR 16, 128, 1
1002 RESTORE_GPR 17, 136, 1
1003 RESTORE_GPR 18, 144, 1
1004 RESTORE_GPR 19, 152, 1
1005 RESTORE_GPR 20, 160, 1
1006 RESTORE_GPR 21, 168, 1
1007 RESTORE_GPR 22, 176, 1
1008 RESTORE_GPR 23, 184, 1
1009 RESTORE_GPR 24, 192, 1
1010 RESTORE_GPR 25, 200, 1
1011 RESTORE_GPR 26, 208, 1
1012 RESTORE_GPR 27, 216, 1
1013 RESTORE_GPR 28, 224, 1
1014 RESTORE_GPR 29, 232, 1
1015 RESTORE_GPR 30, 240, 1
1016 RESTORE_GPR 31, 248, 1
1018 addi 1, 1, 400
1019 ld 0, 16(1)
1020 mtlr 0
1022 blr
1024 Out_no_poly1305_64:
1025 li 3, 0
1026 blr
1027 SYM_FUNC_END(poly1305_64s)
1029 #
1030 # Input: r3 = h, r4 = s, r5 = mac
1031 # mac = h + s
1032 #
1033 SYM_FUNC_START(poly1305_emit_64)
1034 ld 10, 0(3)
1035 ld 11, 8(3)
1036 ld 12, 16(3)
1038 # compare modulus
1039 # h + 5 + (-p)
1040 mr 6, 10
1041 mr 7, 11
1042 mr 8, 12
1043 addic. 6, 6, 5
1044 addze 7, 7
1045 addze 8, 8
1046 srdi 9, 8, 2 # overflow?
1047 cmpdi 9, 0
1048 beq Skip_h64
1049 mr 10, 6
1050 mr 11, 7
1051 mr 12, 8
1053 Skip_h64:
1054 ld 6, 0(4)
1055 ld 7, 8(4)
1056 addc 10, 10, 6
1057 adde 11, 11, 7
1058 addze 12, 12
1060 std 10, 0(5)
1061 std 11, 8(5)
1062 blr
1063 SYM_FUNC_END(poly1305_emit_64)
1065 SYM_DATA_START_LOCAL(RMASK)
1066 .align 5
1067 rmask:
1068 .byte 0xff, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f
1069 cnum:
1070 .long 0x03ffffff, 0x00000000, 0x03ffffff, 0x00000000
1071 .long 0x1a, 0x00, 0x1a, 0x00
1072 .long 0x01000000, 0x01000000, 0x01000000, 0x01000000
1073 .long 0x00010203, 0x04050607, 0x10111213, 0x14151617
1074 .long 0x08090a0b, 0x0c0d0e0f, 0x18191a1b, 0x1c1d1e1f
1075 SYM_DATA_END(RMASK)