?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
WIP FPC-III support
[linux/fpc-iii.git] / drivers / crypto / vmx / aesp8-ppc.pl
blob50a0a18f35da380a8b455ce3dc1f2a54b55fe2ae
1 #! /usr/bin/env perl
2 # SPDX-License-Identifier: GPL-2.0
3
4 # This code is taken from CRYPTOGAMs[1] and is included here using the option
5 # in the license to distribute the code under the GPL. Therefore this program
6 # is free software; you can redistribute it and/or modify it under the terms of
7 # the GNU General Public License version 2 as published by the Free Software
8 # Foundation.
9 #
10 # [1] https://www.openssl.org/~appro/cryptogams/
11
12 # Copyright (c) 2006-2017, CRYPTOGAMS by <appro@openssl.org>
13 # All rights reserved.
14 #
15 # Redistribution and use in source and binary forms, with or without
16 # modification, are permitted provided that the following conditions
17 # are met:
18 #
19 # * Redistributions of source code must retain copyright notices,
20 # this list of conditions and the following disclaimer.
21 #
22 # * Redistributions in binary form must reproduce the above
23 # copyright notice, this list of conditions and the following
24 # disclaimer in the documentation and/or other materials
25 # provided with the distribution.
26 #
27 # * Neither the name of the CRYPTOGAMS nor the names of its
28 # copyright holder and contributors may be used to endorse or
29 # promote products derived from this software without specific
30 # prior written permission.
31 #
32 # ALTERNATIVELY, provided that this notice is retained in full, this
33 # product may be distributed under the terms of the GNU General Public
34 # License (GPL), in which case the provisions of the GPL apply INSTEAD OF
35 # those given above.
36 #
37 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
38 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
39 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
40 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
41 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
42 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
43 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
44 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
45 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
46 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
47 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
48
49 # ====================================================================
50 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
51 # project. The module is, however, dual licensed under OpenSSL and
52 # CRYPTOGAMS licenses depending on where you obtain it. For further
53 # details see https://www.openssl.org/~appro/cryptogams/.
54 # ====================================================================
55 #
56 # This module implements support for AES instructions as per PowerISA
57 # specification version 2.07, first implemented by POWER8 processor.
58 # The module is endian-agnostic in sense that it supports both big-
59 # and little-endian cases. Data alignment in parallelizable modes is
60 # handled with VSX loads and stores, which implies MSR.VSX flag being
61 # set. It should also be noted that ISA specification doesn't prohibit
62 # alignment exceptions for these instructions on page boundaries.
63 # Initially alignment was handled in pure AltiVec/VMX way [when data
64 # is aligned programmatically, which in turn guarantees exception-
65 # free execution], but it turned to hamper performance when vcipher
66 # instructions are interleaved. It's reckoned that eventual
67 # misalignment penalties at page boundaries are in average lower
68 # than additional overhead in pure AltiVec approach.
69 #
70 # May 2016
71 #
72 # Add XTS subroutine, 9x on little- and 12x improvement on big-endian
73 # systems were measured.
74 #
75 ######################################################################
76 # Current large-block performance in cycles per byte processed with
77 # 128-bit key (less is better).
78 #
79 # CBC en-/decrypt CTR XTS
80 # POWER8[le] 3.96/0.72 0.74 1.1
81 # POWER8[be] 3.75/0.65 0.66 1.0
82
83 $flavour = shift;
84
85 if ($flavour =~ /64/) {
86 $SIZE_T =8;
87 $LRSAVE =2*$SIZE_T;
88 $STU ="stdu";
89 $POP ="ld";
90 $PUSH ="std";
91 $UCMP ="cmpld";
92 $SHL ="sldi";
93 } elsif ($flavour =~ /32/) {
94 $SIZE_T =4;
95 $LRSAVE =$SIZE_T;
96 $STU ="stwu";
97 $POP ="lwz";
98 $PUSH ="stw";
99 $UCMP ="cmplw";
100 $SHL ="slwi";
101 } else { die "nonsense $flavour"; }
102
103 $LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;
104
105 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
106 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
107 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
108 die "can't locate ppc-xlate.pl";
109
110 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
111
112 $FRAME=8*$SIZE_T;
113 $prefix="aes_p8";
114
115 $sp="r1";
116 $vrsave="r12";
117
118 #########################################################################
119 {{{ # Key setup procedures #
120 my ($inp,$bits,$out,$ptr,$cnt,$rounds)=map("r$_",(3..8));
121 my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6));
122 my ($stage,$outperm,$outmask,$outhead,$outtail)=map("v$_",(7..11));
123
124 $code.=<<___;
125 .machine "any"
126
127 .text
128
129 .align 7
130 rcon:
131 .long 0x01000000, 0x01000000, 0x01000000, 0x01000000 ?rev
132 .long 0x1b000000, 0x1b000000, 0x1b000000, 0x1b000000 ?rev
133 .long 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c ?rev
134 .long 0,0,0,0 ?asis
135 Lconsts:
136 mflr r0
137 bcl 20,31,\$+4
138 mflr $ptr #vvvvv "distance between . and rcon
139 addi $ptr,$ptr,-0x48
140 mtlr r0
141 blr
142 .long 0
143 .byte 0,12,0x14,0,0,0,0,0
144 .asciz "AES for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
145
146 .globl .${prefix}_set_encrypt_key
147 Lset_encrypt_key:
148 mflr r11
149 $PUSH r11,$LRSAVE($sp)
150
151 li $ptr,-1
152 ${UCMP}i $inp,0
153 beq- Lenc_key_abort # if ($inp==0) return -1;
154 ${UCMP}i $out,0
155 beq- Lenc_key_abort # if ($out==0) return -1;
156 li $ptr,-2
157 cmpwi $bits,128
158 blt- Lenc_key_abort
159 cmpwi $bits,256
160 bgt- Lenc_key_abort
161 andi. r0,$bits,0x3f
162 bne- Lenc_key_abort
163
164 lis r0,0xfff0
165 mfspr $vrsave,256
166 mtspr 256,r0
167
168 bl Lconsts
169 mtlr r11
170
171 neg r9,$inp
172 lvx $in0,0,$inp
173 addi $inp,$inp,15 # 15 is not typo
174 lvsr $key,0,r9 # borrow $key
175 li r8,0x20
176 cmpwi $bits,192
177 lvx $in1,0,$inp
178 le?vspltisb $mask,0x0f # borrow $mask
179 lvx $rcon,0,$ptr
180 le?vxor $key,$key,$mask # adjust for byte swap
181 lvx $mask,r8,$ptr
182 addi $ptr,$ptr,0x10
183 vperm $in0,$in0,$in1,$key # align [and byte swap in LE]
184 li $cnt,8
185 vxor $zero,$zero,$zero
186 mtctr $cnt
187
188 ?lvsr $outperm,0,$out
189 vspltisb $outmask,-1
190 lvx $outhead,0,$out
191 ?vperm $outmask,$zero,$outmask,$outperm
192
193 blt Loop128
194 addi $inp,$inp,8
195 beq L192
196 addi $inp,$inp,8
197 b L256
198
199 .align 4
200 Loop128:
201 vperm $key,$in0,$in0,$mask # rotate-n-splat
202 vsldoi $tmp,$zero,$in0,12 # >>32
203 vperm $outtail,$in0,$in0,$outperm # rotate
204 vsel $stage,$outhead,$outtail,$outmask
205 vmr $outhead,$outtail
206 vcipherlast $key,$key,$rcon
207 stvx $stage,0,$out
208 addi $out,$out,16
209
210 vxor $in0,$in0,$tmp
211 vsldoi $tmp,$zero,$tmp,12 # >>32
212 vxor $in0,$in0,$tmp
213 vsldoi $tmp,$zero,$tmp,12 # >>32
214 vxor $in0,$in0,$tmp
215 vadduwm $rcon,$rcon,$rcon
216 vxor $in0,$in0,$key
217 bdnz Loop128
218
219 lvx $rcon,0,$ptr # last two round keys
220
221 vperm $key,$in0,$in0,$mask # rotate-n-splat
222 vsldoi $tmp,$zero,$in0,12 # >>32
223 vperm $outtail,$in0,$in0,$outperm # rotate
224 vsel $stage,$outhead,$outtail,$outmask
225 vmr $outhead,$outtail
226 vcipherlast $key,$key,$rcon
227 stvx $stage,0,$out
228 addi $out,$out,16
229
230 vxor $in0,$in0,$tmp
231 vsldoi $tmp,$zero,$tmp,12 # >>32
232 vxor $in0,$in0,$tmp
233 vsldoi $tmp,$zero,$tmp,12 # >>32
234 vxor $in0,$in0,$tmp
235 vadduwm $rcon,$rcon,$rcon
236 vxor $in0,$in0,$key
237
238 vperm $key,$in0,$in0,$mask # rotate-n-splat
239 vsldoi $tmp,$zero,$in0,12 # >>32
240 vperm $outtail,$in0,$in0,$outperm # rotate
241 vsel $stage,$outhead,$outtail,$outmask
242 vmr $outhead,$outtail
243 vcipherlast $key,$key,$rcon
244 stvx $stage,0,$out
245 addi $out,$out,16
246
247 vxor $in0,$in0,$tmp
248 vsldoi $tmp,$zero,$tmp,12 # >>32
249 vxor $in0,$in0,$tmp
250 vsldoi $tmp,$zero,$tmp,12 # >>32
251 vxor $in0,$in0,$tmp
252 vxor $in0,$in0,$key
253 vperm $outtail,$in0,$in0,$outperm # rotate
254 vsel $stage,$outhead,$outtail,$outmask
255 vmr $outhead,$outtail
256 stvx $stage,0,$out
257
258 addi $inp,$out,15 # 15 is not typo
259 addi $out,$out,0x50
260
261 li $rounds,10
262 b Ldone
263
264 .align 4
265 L192:
266 lvx $tmp,0,$inp
267 li $cnt,4
268 vperm $outtail,$in0,$in0,$outperm # rotate
269 vsel $stage,$outhead,$outtail,$outmask
270 vmr $outhead,$outtail
271 stvx $stage,0,$out
272 addi $out,$out,16
273 vperm $in1,$in1,$tmp,$key # align [and byte swap in LE]
274 vspltisb $key,8 # borrow $key
275 mtctr $cnt
276 vsububm $mask,$mask,$key # adjust the mask
277
278 Loop192:
279 vperm $key,$in1,$in1,$mask # roate-n-splat
280 vsldoi $tmp,$zero,$in0,12 # >>32
281 vcipherlast $key,$key,$rcon
282
283 vxor $in0,$in0,$tmp
284 vsldoi $tmp,$zero,$tmp,12 # >>32
285 vxor $in0,$in0,$tmp
286 vsldoi $tmp,$zero,$tmp,12 # >>32
287 vxor $in0,$in0,$tmp
288
289 vsldoi $stage,$zero,$in1,8
290 vspltw $tmp,$in0,3
291 vxor $tmp,$tmp,$in1
292 vsldoi $in1,$zero,$in1,12 # >>32
293 vadduwm $rcon,$rcon,$rcon
294 vxor $in1,$in1,$tmp
295 vxor $in0,$in0,$key
296 vxor $in1,$in1,$key
297 vsldoi $stage,$stage,$in0,8
298
299 vperm $key,$in1,$in1,$mask # rotate-n-splat
300 vsldoi $tmp,$zero,$in0,12 # >>32
301 vperm $outtail,$stage,$stage,$outperm # rotate
302 vsel $stage,$outhead,$outtail,$outmask
303 vmr $outhead,$outtail
304 vcipherlast $key,$key,$rcon
305 stvx $stage,0,$out
306 addi $out,$out,16
307
308 vsldoi $stage,$in0,$in1,8
309 vxor $in0,$in0,$tmp
310 vsldoi $tmp,$zero,$tmp,12 # >>32
311 vperm $outtail,$stage,$stage,$outperm # rotate
312 vsel $stage,$outhead,$outtail,$outmask
313 vmr $outhead,$outtail
314 vxor $in0,$in0,$tmp
315 vsldoi $tmp,$zero,$tmp,12 # >>32
316 vxor $in0,$in0,$tmp
317 stvx $stage,0,$out
318 addi $out,$out,16
319
320 vspltw $tmp,$in0,3
321 vxor $tmp,$tmp,$in1
322 vsldoi $in1,$zero,$in1,12 # >>32
323 vadduwm $rcon,$rcon,$rcon
324 vxor $in1,$in1,$tmp
325 vxor $in0,$in0,$key
326 vxor $in1,$in1,$key
327 vperm $outtail,$in0,$in0,$outperm # rotate
328 vsel $stage,$outhead,$outtail,$outmask
329 vmr $outhead,$outtail
330 stvx $stage,0,$out
331 addi $inp,$out,15 # 15 is not typo
332 addi $out,$out,16
333 bdnz Loop192
334
335 li $rounds,12
336 addi $out,$out,0x20
337 b Ldone
338
339 .align 4
340 L256:
341 lvx $tmp,0,$inp
342 li $cnt,7
343 li $rounds,14
344 vperm $outtail,$in0,$in0,$outperm # rotate
345 vsel $stage,$outhead,$outtail,$outmask
346 vmr $outhead,$outtail
347 stvx $stage,0,$out
348 addi $out,$out,16
349 vperm $in1,$in1,$tmp,$key # align [and byte swap in LE]
350 mtctr $cnt
351
352 Loop256:
353 vperm $key,$in1,$in1,$mask # rotate-n-splat
354 vsldoi $tmp,$zero,$in0,12 # >>32
355 vperm $outtail,$in1,$in1,$outperm # rotate
356 vsel $stage,$outhead,$outtail,$outmask
357 vmr $outhead,$outtail
358 vcipherlast $key,$key,$rcon
359 stvx $stage,0,$out
360 addi $out,$out,16
361
362 vxor $in0,$in0,$tmp
363 vsldoi $tmp,$zero,$tmp,12 # >>32
364 vxor $in0,$in0,$tmp
365 vsldoi $tmp,$zero,$tmp,12 # >>32
366 vxor $in0,$in0,$tmp
367 vadduwm $rcon,$rcon,$rcon
368 vxor $in0,$in0,$key
369 vperm $outtail,$in0,$in0,$outperm # rotate
370 vsel $stage,$outhead,$outtail,$outmask
371 vmr $outhead,$outtail
372 stvx $stage,0,$out
373 addi $inp,$out,15 # 15 is not typo
374 addi $out,$out,16
375 bdz Ldone
376
377 vspltw $key,$in0,3 # just splat
378 vsldoi $tmp,$zero,$in1,12 # >>32
379 vsbox $key,$key
380
381 vxor $in1,$in1,$tmp
382 vsldoi $tmp,$zero,$tmp,12 # >>32
383 vxor $in1,$in1,$tmp
384 vsldoi $tmp,$zero,$tmp,12 # >>32
385 vxor $in1,$in1,$tmp
386
387 vxor $in1,$in1,$key
388 b Loop256
389
390 .align 4
391 Ldone:
392 lvx $in1,0,$inp # redundant in aligned case
393 vsel $in1,$outhead,$in1,$outmask
394 stvx $in1,0,$inp
395 li $ptr,0
396 mtspr 256,$vrsave
397 stw $rounds,0($out)
398
399 Lenc_key_abort:
400 mr r3,$ptr
401 blr
402 .long 0
403 .byte 0,12,0x14,1,0,0,3,0
404 .long 0
405 .size .${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key
406
407 .globl .${prefix}_set_decrypt_key
408 $STU $sp,-$FRAME($sp)
409 mflr r10
410 $PUSH r10,$FRAME+$LRSAVE($sp)
411 bl Lset_encrypt_key
412 mtlr r10
413
414 cmpwi r3,0
415 bne- Ldec_key_abort
416
417 slwi $cnt,$rounds,4
418 subi $inp,$out,240 # first round key
419 srwi $rounds,$rounds,1
420 add $out,$inp,$cnt # last round key
421 mtctr $rounds
422
423 Ldeckey:
424 lwz r0, 0($inp)
425 lwz r6, 4($inp)
426 lwz r7, 8($inp)
427 lwz r8, 12($inp)
428 addi $inp,$inp,16
429 lwz r9, 0($out)
430 lwz r10,4($out)
431 lwz r11,8($out)
432 lwz r12,12($out)
433 stw r0, 0($out)
434 stw r6, 4($out)
435 stw r7, 8($out)
436 stw r8, 12($out)
437 subi $out,$out,16
438 stw r9, -16($inp)
439 stw r10,-12($inp)
440 stw r11,-8($inp)
441 stw r12,-4($inp)
442 bdnz Ldeckey
443
444 xor r3,r3,r3 # return value
445 Ldec_key_abort:
446 addi $sp,$sp,$FRAME
447 blr
448 .long 0
449 .byte 0,12,4,1,0x80,0,3,0
450 .long 0
451 .size .${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key
452 ___
453 }}}
454 #########################################################################
455 {{{ # Single block en- and decrypt procedures #
456 sub gen_block () {
457 my $dir = shift;
458 my $n = $dir eq "de" ? "n" : "";
459 my ($inp,$out,$key,$rounds,$idx)=map("r$_",(3..7));
460
461 $code.=<<___;
462 .globl .${prefix}_${dir}crypt
463 lwz $rounds,240($key)
464 lis r0,0xfc00
465 mfspr $vrsave,256
466 li $idx,15 # 15 is not typo
467 mtspr 256,r0
468
469 lvx v0,0,$inp
470 neg r11,$out
471 lvx v1,$idx,$inp
472 lvsl v2,0,$inp # inpperm
473 le?vspltisb v4,0x0f
474 ?lvsl v3,0,r11 # outperm
475 le?vxor v2,v2,v4
476 li $idx,16
477 vperm v0,v0,v1,v2 # align [and byte swap in LE]
478 lvx v1,0,$key
479 ?lvsl v5,0,$key # keyperm
480 srwi $rounds,$rounds,1
481 lvx v2,$idx,$key
482 addi $idx,$idx,16
483 subi $rounds,$rounds,1
484 ?vperm v1,v1,v2,v5 # align round key
485
486 vxor v0,v0,v1
487 lvx v1,$idx,$key
488 addi $idx,$idx,16
489 mtctr $rounds
490
491 Loop_${dir}c:
492 ?vperm v2,v2,v1,v5
493 v${n}cipher v0,v0,v2
494 lvx v2,$idx,$key
495 addi $idx,$idx,16
496 ?vperm v1,v1,v2,v5
497 v${n}cipher v0,v0,v1
498 lvx v1,$idx,$key
499 addi $idx,$idx,16
500 bdnz Loop_${dir}c
501
502 ?vperm v2,v2,v1,v5
503 v${n}cipher v0,v0,v2
504 lvx v2,$idx,$key
505 ?vperm v1,v1,v2,v5
506 v${n}cipherlast v0,v0,v1
507
508 vspltisb v2,-1
509 vxor v1,v1,v1
510 li $idx,15 # 15 is not typo
511 ?vperm v2,v1,v2,v3 # outmask
512 le?vxor v3,v3,v4
513 lvx v1,0,$out # outhead
514 vperm v0,v0,v0,v3 # rotate [and byte swap in LE]
515 vsel v1,v1,v0,v2
516 lvx v4,$idx,$out
517 stvx v1,0,$out
518 vsel v0,v0,v4,v2
519 stvx v0,$idx,$out
520
521 mtspr 256,$vrsave
522 blr
523 .long 0
524 .byte 0,12,0x14,0,0,0,3,0
525 .long 0
526 .size .${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt
527 ___
528 }
529 &gen_block("en");
530 &gen_block("de");
531 }}}
532 #########################################################################
533 {{{ # CBC en- and decrypt procedures #
534 my ($inp,$out,$len,$key,$ivp,$enc,$rounds,$idx)=map("r$_",(3..10));
535 my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3));
536 my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)=
537 map("v$_",(4..10));
538 $code.=<<___;
539 .globl .${prefix}_cbc_encrypt
540 ${UCMP}i $len,16
541 bltlr-
542
543 cmpwi $enc,0 # test direction
544 lis r0,0xffe0
545 mfspr $vrsave,256
546 mtspr 256,r0
547
548 li $idx,15
549 vxor $rndkey0,$rndkey0,$rndkey0
550 le?vspltisb $tmp,0x0f
551
552 lvx $ivec,0,$ivp # load [unaligned] iv
553 lvsl $inpperm,0,$ivp
554 lvx $inptail,$idx,$ivp
555 le?vxor $inpperm,$inpperm,$tmp
556 vperm $ivec,$ivec,$inptail,$inpperm
557
558 neg r11,$inp
559 ?lvsl $keyperm,0,$key # prepare for unaligned key
560 lwz $rounds,240($key)
561
562 lvsr $inpperm,0,r11 # prepare for unaligned load
563 lvx $inptail,0,$inp
564 addi $inp,$inp,15 # 15 is not typo
565 le?vxor $inpperm,$inpperm,$tmp
566
567 ?lvsr $outperm,0,$out # prepare for unaligned store
568 vspltisb $outmask,-1
569 lvx $outhead,0,$out
570 ?vperm $outmask,$rndkey0,$outmask,$outperm
571 le?vxor $outperm,$outperm,$tmp
572
573 srwi $rounds,$rounds,1
574 li $idx,16
575 subi $rounds,$rounds,1
576 beq Lcbc_dec
577
578 Lcbc_enc:
579 vmr $inout,$inptail
580 lvx $inptail,0,$inp
581 addi $inp,$inp,16
582 mtctr $rounds
583 subi $len,$len,16 # len-=16
584
585 lvx $rndkey0,0,$key
586 vperm $inout,$inout,$inptail,$inpperm
587 lvx $rndkey1,$idx,$key
588 addi $idx,$idx,16
589 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
590 vxor $inout,$inout,$rndkey0
591 lvx $rndkey0,$idx,$key
592 addi $idx,$idx,16
593 vxor $inout,$inout,$ivec
594
595 Loop_cbc_enc:
596 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
597 vcipher $inout,$inout,$rndkey1
598 lvx $rndkey1,$idx,$key
599 addi $idx,$idx,16
600 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
601 vcipher $inout,$inout,$rndkey0
602 lvx $rndkey0,$idx,$key
603 addi $idx,$idx,16
604 bdnz Loop_cbc_enc
605
606 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
607 vcipher $inout,$inout,$rndkey1
608 lvx $rndkey1,$idx,$key
609 li $idx,16
610 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
611 vcipherlast $ivec,$inout,$rndkey0
612 ${UCMP}i $len,16
613
614 vperm $tmp,$ivec,$ivec,$outperm
615 vsel $inout,$outhead,$tmp,$outmask
616 vmr $outhead,$tmp
617 stvx $inout,0,$out
618 addi $out,$out,16
619 bge Lcbc_enc
620
621 b Lcbc_done
622
623 .align 4
624 Lcbc_dec:
625 ${UCMP}i $len,128
626 bge _aesp8_cbc_decrypt8x
627 vmr $tmp,$inptail
628 lvx $inptail,0,$inp
629 addi $inp,$inp,16
630 mtctr $rounds
631 subi $len,$len,16 # len-=16
632
633 lvx $rndkey0,0,$key
634 vperm $tmp,$tmp,$inptail,$inpperm
635 lvx $rndkey1,$idx,$key
636 addi $idx,$idx,16
637 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
638 vxor $inout,$tmp,$rndkey0
639 lvx $rndkey0,$idx,$key
640 addi $idx,$idx,16
641
642 Loop_cbc_dec:
643 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
644 vncipher $inout,$inout,$rndkey1
645 lvx $rndkey1,$idx,$key
646 addi $idx,$idx,16
647 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
648 vncipher $inout,$inout,$rndkey0
649 lvx $rndkey0,$idx,$key
650 addi $idx,$idx,16
651 bdnz Loop_cbc_dec
652
653 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
654 vncipher $inout,$inout,$rndkey1
655 lvx $rndkey1,$idx,$key
656 li $idx,16
657 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
658 vncipherlast $inout,$inout,$rndkey0
659 ${UCMP}i $len,16
660
661 vxor $inout,$inout,$ivec
662 vmr $ivec,$tmp
663 vperm $tmp,$inout,$inout,$outperm
664 vsel $inout,$outhead,$tmp,$outmask
665 vmr $outhead,$tmp
666 stvx $inout,0,$out
667 addi $out,$out,16
668 bge Lcbc_dec
669
670 Lcbc_done:
671 addi $out,$out,-1
672 lvx $inout,0,$out # redundant in aligned case
673 vsel $inout,$outhead,$inout,$outmask
674 stvx $inout,0,$out
675
676 neg $enc,$ivp # write [unaligned] iv
677 li $idx,15 # 15 is not typo
678 vxor $rndkey0,$rndkey0,$rndkey0
679 vspltisb $outmask,-1
680 le?vspltisb $tmp,0x0f
681 ?lvsl $outperm,0,$enc
682 ?vperm $outmask,$rndkey0,$outmask,$outperm
683 le?vxor $outperm,$outperm,$tmp
684 lvx $outhead,0,$ivp
685 vperm $ivec,$ivec,$ivec,$outperm
686 vsel $inout,$outhead,$ivec,$outmask
687 lvx $inptail,$idx,$ivp
688 stvx $inout,0,$ivp
689 vsel $inout,$ivec,$inptail,$outmask
690 stvx $inout,$idx,$ivp
691
692 mtspr 256,$vrsave
693 blr
694 .long 0
695 .byte 0,12,0x14,0,0,0,6,0
696 .long 0
697 ___
698 #########################################################################
699 {{ # Optimized CBC decrypt procedure #
700 my $key_="r11";
701 my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
702 my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13));
703 my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(14..21));
704 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
705 # v26-v31 last 6 round keys
706 my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment
707
708 $code.=<<___;
709 .align 5
710 _aesp8_cbc_decrypt8x:
711 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
712 li r10,`$FRAME+8*16+15`
713 li r11,`$FRAME+8*16+31`
714 stvx v20,r10,$sp # ABI says so
715 addi r10,r10,32
716 stvx v21,r11,$sp
717 addi r11,r11,32
718 stvx v22,r10,$sp
719 addi r10,r10,32
720 stvx v23,r11,$sp
721 addi r11,r11,32
722 stvx v24,r10,$sp
723 addi r10,r10,32
724 stvx v25,r11,$sp
725 addi r11,r11,32
726 stvx v26,r10,$sp
727 addi r10,r10,32
728 stvx v27,r11,$sp
729 addi r11,r11,32
730 stvx v28,r10,$sp
731 addi r10,r10,32
732 stvx v29,r11,$sp
733 addi r11,r11,32
734 stvx v30,r10,$sp
735 stvx v31,r11,$sp
736 li r0,-1
737 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
738 li $x10,0x10
739 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
740 li $x20,0x20
741 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
742 li $x30,0x30
743 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
744 li $x40,0x40
745 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
746 li $x50,0x50
747 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
748 li $x60,0x60
749 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
750 li $x70,0x70
751 mtspr 256,r0
752
753 subi $rounds,$rounds,3 # -4 in total
754 subi $len,$len,128 # bias
755
756 lvx $rndkey0,$x00,$key # load key schedule
757 lvx v30,$x10,$key
758 addi $key,$key,0x20
759 lvx v31,$x00,$key
760 ?vperm $rndkey0,$rndkey0,v30,$keyperm
761 addi $key_,$sp,$FRAME+15
762 mtctr $rounds
763
764 Load_cbc_dec_key:
765 ?vperm v24,v30,v31,$keyperm
766 lvx v30,$x10,$key
767 addi $key,$key,0x20
768 stvx v24,$x00,$key_ # off-load round[1]
769 ?vperm v25,v31,v30,$keyperm
770 lvx v31,$x00,$key
771 stvx v25,$x10,$key_ # off-load round[2]
772 addi $key_,$key_,0x20
773 bdnz Load_cbc_dec_key
774
775 lvx v26,$x10,$key
776 ?vperm v24,v30,v31,$keyperm
777 lvx v27,$x20,$key
778 stvx v24,$x00,$key_ # off-load round[3]
779 ?vperm v25,v31,v26,$keyperm
780 lvx v28,$x30,$key
781 stvx v25,$x10,$key_ # off-load round[4]
782 addi $key_,$sp,$FRAME+15 # rewind $key_
783 ?vperm v26,v26,v27,$keyperm
784 lvx v29,$x40,$key
785 ?vperm v27,v27,v28,$keyperm
786 lvx v30,$x50,$key
787 ?vperm v28,v28,v29,$keyperm
788 lvx v31,$x60,$key
789 ?vperm v29,v29,v30,$keyperm
790 lvx $out0,$x70,$key # borrow $out0
791 ?vperm v30,v30,v31,$keyperm
792 lvx v24,$x00,$key_ # pre-load round[1]
793 ?vperm v31,v31,$out0,$keyperm
794 lvx v25,$x10,$key_ # pre-load round[2]
795
796 #lvx $inptail,0,$inp # "caller" already did this
797 #addi $inp,$inp,15 # 15 is not typo
798 subi $inp,$inp,15 # undo "caller"
799
800 le?li $idx,8
801 lvx_u $in0,$x00,$inp # load first 8 "words"
802 le?lvsl $inpperm,0,$idx
803 le?vspltisb $tmp,0x0f
804 lvx_u $in1,$x10,$inp
805 le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u
806 lvx_u $in2,$x20,$inp
807 le?vperm $in0,$in0,$in0,$inpperm
808 lvx_u $in3,$x30,$inp
809 le?vperm $in1,$in1,$in1,$inpperm
810 lvx_u $in4,$x40,$inp
811 le?vperm $in2,$in2,$in2,$inpperm
812 vxor $out0,$in0,$rndkey0
813 lvx_u $in5,$x50,$inp
814 le?vperm $in3,$in3,$in3,$inpperm
815 vxor $out1,$in1,$rndkey0
816 lvx_u $in6,$x60,$inp
817 le?vperm $in4,$in4,$in4,$inpperm
818 vxor $out2,$in2,$rndkey0
819 lvx_u $in7,$x70,$inp
820 addi $inp,$inp,0x80
821 le?vperm $in5,$in5,$in5,$inpperm
822 vxor $out3,$in3,$rndkey0
823 le?vperm $in6,$in6,$in6,$inpperm
824 vxor $out4,$in4,$rndkey0
825 le?vperm $in7,$in7,$in7,$inpperm
826 vxor $out5,$in5,$rndkey0
827 vxor $out6,$in6,$rndkey0
828 vxor $out7,$in7,$rndkey0
829
830 mtctr $rounds
831 b Loop_cbc_dec8x
832 .align 5
833 Loop_cbc_dec8x:
834 vncipher $out0,$out0,v24
835 vncipher $out1,$out1,v24
836 vncipher $out2,$out2,v24
837 vncipher $out3,$out3,v24
838 vncipher $out4,$out4,v24
839 vncipher $out5,$out5,v24
840 vncipher $out6,$out6,v24
841 vncipher $out7,$out7,v24
842 lvx v24,$x20,$key_ # round[3]
843 addi $key_,$key_,0x20
844
845 vncipher $out0,$out0,v25
846 vncipher $out1,$out1,v25
847 vncipher $out2,$out2,v25
848 vncipher $out3,$out3,v25
849 vncipher $out4,$out4,v25
850 vncipher $out5,$out5,v25
851 vncipher $out6,$out6,v25
852 vncipher $out7,$out7,v25
853 lvx v25,$x10,$key_ # round[4]
854 bdnz Loop_cbc_dec8x
855
856 subic $len,$len,128 # $len-=128
857 vncipher $out0,$out0,v24
858 vncipher $out1,$out1,v24
859 vncipher $out2,$out2,v24
860 vncipher $out3,$out3,v24
861 vncipher $out4,$out4,v24
862 vncipher $out5,$out5,v24
863 vncipher $out6,$out6,v24
864 vncipher $out7,$out7,v24
865
866 subfe. r0,r0,r0 # borrow?-1:0
867 vncipher $out0,$out0,v25
868 vncipher $out1,$out1,v25
869 vncipher $out2,$out2,v25
870 vncipher $out3,$out3,v25
871 vncipher $out4,$out4,v25
872 vncipher $out5,$out5,v25
873 vncipher $out6,$out6,v25
874 vncipher $out7,$out7,v25
875
876 and r0,r0,$len
877 vncipher $out0,$out0,v26
878 vncipher $out1,$out1,v26
879 vncipher $out2,$out2,v26
880 vncipher $out3,$out3,v26
881 vncipher $out4,$out4,v26
882 vncipher $out5,$out5,v26
883 vncipher $out6,$out6,v26
884 vncipher $out7,$out7,v26
885
886 add $inp,$inp,r0 # $inp is adjusted in such
887 # way that at exit from the
888 # loop inX-in7 are loaded
889 # with last "words"
890 vncipher $out0,$out0,v27
891 vncipher $out1,$out1,v27
892 vncipher $out2,$out2,v27
893 vncipher $out3,$out3,v27
894 vncipher $out4,$out4,v27
895 vncipher $out5,$out5,v27
896 vncipher $out6,$out6,v27
897 vncipher $out7,$out7,v27
898
899 addi $key_,$sp,$FRAME+15 # rewind $key_
900 vncipher $out0,$out0,v28
901 vncipher $out1,$out1,v28
902 vncipher $out2,$out2,v28
903 vncipher $out3,$out3,v28
904 vncipher $out4,$out4,v28
905 vncipher $out5,$out5,v28
906 vncipher $out6,$out6,v28
907 vncipher $out7,$out7,v28
908 lvx v24,$x00,$key_ # re-pre-load round[1]
909
910 vncipher $out0,$out0,v29
911 vncipher $out1,$out1,v29
912 vncipher $out2,$out2,v29
913 vncipher $out3,$out3,v29
914 vncipher $out4,$out4,v29
915 vncipher $out5,$out5,v29
916 vncipher $out6,$out6,v29
917 vncipher $out7,$out7,v29
918 lvx v25,$x10,$key_ # re-pre-load round[2]
919
920 vncipher $out0,$out0,v30
921 vxor $ivec,$ivec,v31 # xor with last round key
922 vncipher $out1,$out1,v30
923 vxor $in0,$in0,v31
924 vncipher $out2,$out2,v30
925 vxor $in1,$in1,v31
926 vncipher $out3,$out3,v30
927 vxor $in2,$in2,v31
928 vncipher $out4,$out4,v30
929 vxor $in3,$in3,v31
930 vncipher $out5,$out5,v30
931 vxor $in4,$in4,v31
932 vncipher $out6,$out6,v30
933 vxor $in5,$in5,v31
934 vncipher $out7,$out7,v30
935 vxor $in6,$in6,v31
936
937 vncipherlast $out0,$out0,$ivec
938 vncipherlast $out1,$out1,$in0
939 lvx_u $in0,$x00,$inp # load next input block
940 vncipherlast $out2,$out2,$in1
941 lvx_u $in1,$x10,$inp
942 vncipherlast $out3,$out3,$in2
943 le?vperm $in0,$in0,$in0,$inpperm
944 lvx_u $in2,$x20,$inp
945 vncipherlast $out4,$out4,$in3
946 le?vperm $in1,$in1,$in1,$inpperm
947 lvx_u $in3,$x30,$inp
948 vncipherlast $out5,$out5,$in4
949 le?vperm $in2,$in2,$in2,$inpperm
950 lvx_u $in4,$x40,$inp
951 vncipherlast $out6,$out6,$in5
952 le?vperm $in3,$in3,$in3,$inpperm
953 lvx_u $in5,$x50,$inp
954 vncipherlast $out7,$out7,$in6
955 le?vperm $in4,$in4,$in4,$inpperm
956 lvx_u $in6,$x60,$inp
957 vmr $ivec,$in7
958 le?vperm $in5,$in5,$in5,$inpperm
959 lvx_u $in7,$x70,$inp
960 addi $inp,$inp,0x80
961
962 le?vperm $out0,$out0,$out0,$inpperm
963 le?vperm $out1,$out1,$out1,$inpperm
964 stvx_u $out0,$x00,$out
965 le?vperm $in6,$in6,$in6,$inpperm
966 vxor $out0,$in0,$rndkey0
967 le?vperm $out2,$out2,$out2,$inpperm
968 stvx_u $out1,$x10,$out
969 le?vperm $in7,$in7,$in7,$inpperm
970 vxor $out1,$in1,$rndkey0
971 le?vperm $out3,$out3,$out3,$inpperm
972 stvx_u $out2,$x20,$out
973 vxor $out2,$in2,$rndkey0
974 le?vperm $out4,$out4,$out4,$inpperm
975 stvx_u $out3,$x30,$out
976 vxor $out3,$in3,$rndkey0
977 le?vperm $out5,$out5,$out5,$inpperm
978 stvx_u $out4,$x40,$out
979 vxor $out4,$in4,$rndkey0
980 le?vperm $out6,$out6,$out6,$inpperm
981 stvx_u $out5,$x50,$out
982 vxor $out5,$in5,$rndkey0
983 le?vperm $out7,$out7,$out7,$inpperm
984 stvx_u $out6,$x60,$out
985 vxor $out6,$in6,$rndkey0
986 stvx_u $out7,$x70,$out
987 addi $out,$out,0x80
988 vxor $out7,$in7,$rndkey0
989
990 mtctr $rounds
991 beq Loop_cbc_dec8x # did $len-=128 borrow?
992
993 addic. $len,$len,128
994 beq Lcbc_dec8x_done
995 nop
996 nop
997
998 Loop_cbc_dec8x_tail: # up to 7 "words" tail...
999 vncipher $out1,$out1,v24
1000 vncipher $out2,$out2,v24
1001 vncipher $out3,$out3,v24
1002 vncipher $out4,$out4,v24
1003 vncipher $out5,$out5,v24
1004 vncipher $out6,$out6,v24
1005 vncipher $out7,$out7,v24
1006 lvx v24,$x20,$key_ # round[3]
1007 addi $key_,$key_,0x20
1008
1009 vncipher $out1,$out1,v25
1010 vncipher $out2,$out2,v25
1011 vncipher $out3,$out3,v25
1012 vncipher $out4,$out4,v25
1013 vncipher $out5,$out5,v25
1014 vncipher $out6,$out6,v25
1015 vncipher $out7,$out7,v25
1016 lvx v25,$x10,$key_ # round[4]
1017 bdnz Loop_cbc_dec8x_tail
1018
1019 vncipher $out1,$out1,v24
1020 vncipher $out2,$out2,v24
1021 vncipher $out3,$out3,v24
1022 vncipher $out4,$out4,v24
1023 vncipher $out5,$out5,v24
1024 vncipher $out6,$out6,v24
1025 vncipher $out7,$out7,v24
1026
1027 vncipher $out1,$out1,v25
1028 vncipher $out2,$out2,v25
1029 vncipher $out3,$out3,v25
1030 vncipher $out4,$out4,v25
1031 vncipher $out5,$out5,v25
1032 vncipher $out6,$out6,v25
1033 vncipher $out7,$out7,v25
1034
1035 vncipher $out1,$out1,v26
1036 vncipher $out2,$out2,v26
1037 vncipher $out3,$out3,v26
1038 vncipher $out4,$out4,v26
1039 vncipher $out5,$out5,v26
1040 vncipher $out6,$out6,v26
1041 vncipher $out7,$out7,v26
1042
1043 vncipher $out1,$out1,v27
1044 vncipher $out2,$out2,v27
1045 vncipher $out3,$out3,v27
1046 vncipher $out4,$out4,v27
1047 vncipher $out5,$out5,v27
1048 vncipher $out6,$out6,v27
1049 vncipher $out7,$out7,v27
1050
1051 vncipher $out1,$out1,v28
1052 vncipher $out2,$out2,v28
1053 vncipher $out3,$out3,v28
1054 vncipher $out4,$out4,v28
1055 vncipher $out5,$out5,v28
1056 vncipher $out6,$out6,v28
1057 vncipher $out7,$out7,v28
1058
1059 vncipher $out1,$out1,v29
1060 vncipher $out2,$out2,v29
1061 vncipher $out3,$out3,v29
1062 vncipher $out4,$out4,v29
1063 vncipher $out5,$out5,v29
1064 vncipher $out6,$out6,v29
1065 vncipher $out7,$out7,v29
1066
1067 vncipher $out1,$out1,v30
1068 vxor $ivec,$ivec,v31 # last round key
1069 vncipher $out2,$out2,v30
1070 vxor $in1,$in1,v31
1071 vncipher $out3,$out3,v30
1072 vxor $in2,$in2,v31
1073 vncipher $out4,$out4,v30
1074 vxor $in3,$in3,v31
1075 vncipher $out5,$out5,v30
1076 vxor $in4,$in4,v31
1077 vncipher $out6,$out6,v30
1078 vxor $in5,$in5,v31
1079 vncipher $out7,$out7,v30
1080 vxor $in6,$in6,v31
1081
1082 cmplwi $len,32 # switch($len)
1083 blt Lcbc_dec8x_one
1084 nop
1085 beq Lcbc_dec8x_two
1086 cmplwi $len,64
1087 blt Lcbc_dec8x_three
1088 nop
1089 beq Lcbc_dec8x_four
1090 cmplwi $len,96
1091 blt Lcbc_dec8x_five
1092 nop
1093 beq Lcbc_dec8x_six
1094
1095 Lcbc_dec8x_seven:
1096 vncipherlast $out1,$out1,$ivec
1097 vncipherlast $out2,$out2,$in1
1098 vncipherlast $out3,$out3,$in2
1099 vncipherlast $out4,$out4,$in3
1100 vncipherlast $out5,$out5,$in4
1101 vncipherlast $out6,$out6,$in5
1102 vncipherlast $out7,$out7,$in6
1103 vmr $ivec,$in7
1104
1105 le?vperm $out1,$out1,$out1,$inpperm
1106 le?vperm $out2,$out2,$out2,$inpperm
1107 stvx_u $out1,$x00,$out
1108 le?vperm $out3,$out3,$out3,$inpperm
1109 stvx_u $out2,$x10,$out
1110 le?vperm $out4,$out4,$out4,$inpperm
1111 stvx_u $out3,$x20,$out
1112 le?vperm $out5,$out5,$out5,$inpperm
1113 stvx_u $out4,$x30,$out
1114 le?vperm $out6,$out6,$out6,$inpperm
1115 stvx_u $out5,$x40,$out
1116 le?vperm $out7,$out7,$out7,$inpperm
1117 stvx_u $out6,$x50,$out
1118 stvx_u $out7,$x60,$out
1119 addi $out,$out,0x70
1120 b Lcbc_dec8x_done
1121
1122 .align 5
1123 Lcbc_dec8x_six:
1124 vncipherlast $out2,$out2,$ivec
1125 vncipherlast $out3,$out3,$in2
1126 vncipherlast $out4,$out4,$in3
1127 vncipherlast $out5,$out5,$in4
1128 vncipherlast $out6,$out6,$in5
1129 vncipherlast $out7,$out7,$in6
1130 vmr $ivec,$in7
1131
1132 le?vperm $out2,$out2,$out2,$inpperm
1133 le?vperm $out3,$out3,$out3,$inpperm
1134 stvx_u $out2,$x00,$out
1135 le?vperm $out4,$out4,$out4,$inpperm
1136 stvx_u $out3,$x10,$out
1137 le?vperm $out5,$out5,$out5,$inpperm
1138 stvx_u $out4,$x20,$out
1139 le?vperm $out6,$out6,$out6,$inpperm
1140 stvx_u $out5,$x30,$out
1141 le?vperm $out7,$out7,$out7,$inpperm
1142 stvx_u $out6,$x40,$out
1143 stvx_u $out7,$x50,$out
1144 addi $out,$out,0x60
1145 b Lcbc_dec8x_done
1146
1147 .align 5
1148 Lcbc_dec8x_five:
1149 vncipherlast $out3,$out3,$ivec
1150 vncipherlast $out4,$out4,$in3
1151 vncipherlast $out5,$out5,$in4
1152 vncipherlast $out6,$out6,$in5
1153 vncipherlast $out7,$out7,$in6
1154 vmr $ivec,$in7
1155
1156 le?vperm $out3,$out3,$out3,$inpperm
1157 le?vperm $out4,$out4,$out4,$inpperm
1158 stvx_u $out3,$x00,$out
1159 le?vperm $out5,$out5,$out5,$inpperm
1160 stvx_u $out4,$x10,$out
1161 le?vperm $out6,$out6,$out6,$inpperm
1162 stvx_u $out5,$x20,$out
1163 le?vperm $out7,$out7,$out7,$inpperm
1164 stvx_u $out6,$x30,$out
1165 stvx_u $out7,$x40,$out
1166 addi $out,$out,0x50
1167 b Lcbc_dec8x_done
1168
1169 .align 5
1170 Lcbc_dec8x_four:
1171 vncipherlast $out4,$out4,$ivec
1172 vncipherlast $out5,$out5,$in4
1173 vncipherlast $out6,$out6,$in5
1174 vncipherlast $out7,$out7,$in6
1175 vmr $ivec,$in7
1176
1177 le?vperm $out4,$out4,$out4,$inpperm
1178 le?vperm $out5,$out5,$out5,$inpperm
1179 stvx_u $out4,$x00,$out
1180 le?vperm $out6,$out6,$out6,$inpperm
1181 stvx_u $out5,$x10,$out
1182 le?vperm $out7,$out7,$out7,$inpperm
1183 stvx_u $out6,$x20,$out
1184 stvx_u $out7,$x30,$out
1185 addi $out,$out,0x40
1186 b Lcbc_dec8x_done
1187
1188 .align 5
1189 Lcbc_dec8x_three:
1190 vncipherlast $out5,$out5,$ivec
1191 vncipherlast $out6,$out6,$in5
1192 vncipherlast $out7,$out7,$in6
1193 vmr $ivec,$in7
1194
1195 le?vperm $out5,$out5,$out5,$inpperm
1196 le?vperm $out6,$out6,$out6,$inpperm
1197 stvx_u $out5,$x00,$out
1198 le?vperm $out7,$out7,$out7,$inpperm
1199 stvx_u $out6,$x10,$out
1200 stvx_u $out7,$x20,$out
1201 addi $out,$out,0x30
1202 b Lcbc_dec8x_done
1203
1204 .align 5
1205 Lcbc_dec8x_two:
1206 vncipherlast $out6,$out6,$ivec
1207 vncipherlast $out7,$out7,$in6
1208 vmr $ivec,$in7
1209
1210 le?vperm $out6,$out6,$out6,$inpperm
1211 le?vperm $out7,$out7,$out7,$inpperm
1212 stvx_u $out6,$x00,$out
1213 stvx_u $out7,$x10,$out
1214 addi $out,$out,0x20
1215 b Lcbc_dec8x_done
1216
1217 .align 5
1218 Lcbc_dec8x_one:
1219 vncipherlast $out7,$out7,$ivec
1220 vmr $ivec,$in7
1221
1222 le?vperm $out7,$out7,$out7,$inpperm
1223 stvx_u $out7,0,$out
1224 addi $out,$out,0x10
1225
1226 Lcbc_dec8x_done:
1227 le?vperm $ivec,$ivec,$ivec,$inpperm
1228 stvx_u $ivec,0,$ivp # write [unaligned] iv
1229
1230 li r10,`$FRAME+15`
1231 li r11,`$FRAME+31`
1232 stvx $inpperm,r10,$sp # wipe copies of round keys
1233 addi r10,r10,32
1234 stvx $inpperm,r11,$sp
1235 addi r11,r11,32
1236 stvx $inpperm,r10,$sp
1237 addi r10,r10,32
1238 stvx $inpperm,r11,$sp
1239 addi r11,r11,32
1240 stvx $inpperm,r10,$sp
1241 addi r10,r10,32
1242 stvx $inpperm,r11,$sp
1243 addi r11,r11,32
1244 stvx $inpperm,r10,$sp
1245 addi r10,r10,32
1246 stvx $inpperm,r11,$sp
1247 addi r11,r11,32
1248
1249 mtspr 256,$vrsave
1250 lvx v20,r10,$sp # ABI says so
1251 addi r10,r10,32
1252 lvx v21,r11,$sp
1253 addi r11,r11,32
1254 lvx v22,r10,$sp
1255 addi r10,r10,32
1256 lvx v23,r11,$sp
1257 addi r11,r11,32
1258 lvx v24,r10,$sp
1259 addi r10,r10,32
1260 lvx v25,r11,$sp
1261 addi r11,r11,32
1262 lvx v26,r10,$sp
1263 addi r10,r10,32
1264 lvx v27,r11,$sp
1265 addi r11,r11,32
1266 lvx v28,r10,$sp
1267 addi r10,r10,32
1268 lvx v29,r11,$sp
1269 addi r11,r11,32
1270 lvx v30,r10,$sp
1271 lvx v31,r11,$sp
1272 $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1273 $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1274 $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1275 $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1276 $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1277 $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1278 addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
1279 blr
1280 .long 0
1281 .byte 0,12,0x14,0,0x80,6,6,0
1282 .long 0
1283 .size .${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt
1284 ___
1285 }} }}}
1286
1287 #########################################################################
1288 {{{ # CTR procedure[s] #
1289
1290 ####################### WARNING: Here be dragons! #######################
1291 #
1292 # This code is written as 'ctr32', based on a 32-bit counter used
1293 # upstream. The kernel does *not* use a 32-bit counter. The kernel uses
1294 # a 128-bit counter.
1295 #
1296 # This leads to subtle changes from the upstream code: the counter
1297 # is incremented with vaddu_q_m rather than vaddu_w_m. This occurs in
1298 # both the bulk (8 blocks at a time) path, and in the individual block
1299 # path. Be aware of this when doing updates.
1300 #
1301 # See:
1302 # 1d4aa0b4c181 ("crypto: vmx - Fixing AES-CTR counter bug")
1303 # 009b30ac7444 ("crypto: vmx - CTR: always increment IV as quadword")
1304 # https://github.com/openssl/openssl/pull/8942
1305 #
1306 #########################################################################
1307 my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10));
1308 my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3));
1309 my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)=
1310 map("v$_",(4..11));
1311 my $dat=$tmp;
1312
1313 $code.=<<___;
1314 .globl .${prefix}_ctr32_encrypt_blocks
1315 ${UCMP}i $len,1
1316 bltlr-
1317
1318 lis r0,0xfff0
1319 mfspr $vrsave,256
1320 mtspr 256,r0
1321
1322 li $idx,15
1323 vxor $rndkey0,$rndkey0,$rndkey0
1324 le?vspltisb $tmp,0x0f
1325
1326 lvx $ivec,0,$ivp # load [unaligned] iv
1327 lvsl $inpperm,0,$ivp
1328 lvx $inptail,$idx,$ivp
1329 vspltisb $one,1
1330 le?vxor $inpperm,$inpperm,$tmp
1331 vperm $ivec,$ivec,$inptail,$inpperm
1332 vsldoi $one,$rndkey0,$one,1
1333
1334 neg r11,$inp
1335 ?lvsl $keyperm,0,$key # prepare for unaligned key
1336 lwz $rounds,240($key)
1337
1338 lvsr $inpperm,0,r11 # prepare for unaligned load
1339 lvx $inptail,0,$inp
1340 addi $inp,$inp,15 # 15 is not typo
1341 le?vxor $inpperm,$inpperm,$tmp
1342
1343 srwi $rounds,$rounds,1
1344 li $idx,16
1345 subi $rounds,$rounds,1
1346
1347 ${UCMP}i $len,8
1348 bge _aesp8_ctr32_encrypt8x
1349
1350 ?lvsr $outperm,0,$out # prepare for unaligned store
1351 vspltisb $outmask,-1
1352 lvx $outhead,0,$out
1353 ?vperm $outmask,$rndkey0,$outmask,$outperm
1354 le?vxor $outperm,$outperm,$tmp
1355
1356 lvx $rndkey0,0,$key
1357 mtctr $rounds
1358 lvx $rndkey1,$idx,$key
1359 addi $idx,$idx,16
1360 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1361 vxor $inout,$ivec,$rndkey0
1362 lvx $rndkey0,$idx,$key
1363 addi $idx,$idx,16
1364 b Loop_ctr32_enc
1365
1366 .align 5
1367 Loop_ctr32_enc:
1368 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
1369 vcipher $inout,$inout,$rndkey1
1370 lvx $rndkey1,$idx,$key
1371 addi $idx,$idx,16
1372 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1373 vcipher $inout,$inout,$rndkey0
1374 lvx $rndkey0,$idx,$key
1375 addi $idx,$idx,16
1376 bdnz Loop_ctr32_enc
1377
1378 vadduqm $ivec,$ivec,$one # Kernel change for 128-bit
1379 vmr $dat,$inptail
1380 lvx $inptail,0,$inp
1381 addi $inp,$inp,16
1382 subic. $len,$len,1 # blocks--
1383
1384 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
1385 vcipher $inout,$inout,$rndkey1
1386 lvx $rndkey1,$idx,$key
1387 vperm $dat,$dat,$inptail,$inpperm
1388 li $idx,16
1389 ?vperm $rndkey1,$rndkey0,$rndkey1,$keyperm
1390 lvx $rndkey0,0,$key
1391 vxor $dat,$dat,$rndkey1 # last round key
1392 vcipherlast $inout,$inout,$dat
1393
1394 lvx $rndkey1,$idx,$key
1395 addi $idx,$idx,16
1396 vperm $inout,$inout,$inout,$outperm
1397 vsel $dat,$outhead,$inout,$outmask
1398 mtctr $rounds
1399 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1400 vmr $outhead,$inout
1401 vxor $inout,$ivec,$rndkey0
1402 lvx $rndkey0,$idx,$key
1403 addi $idx,$idx,16
1404 stvx $dat,0,$out
1405 addi $out,$out,16
1406 bne Loop_ctr32_enc
1407
1408 addi $out,$out,-1
1409 lvx $inout,0,$out # redundant in aligned case
1410 vsel $inout,$outhead,$inout,$outmask
1411 stvx $inout,0,$out
1412
1413 mtspr 256,$vrsave
1414 blr
1415 .long 0
1416 .byte 0,12,0x14,0,0,0,6,0
1417 .long 0
1418 ___
1419 #########################################################################
1420 {{ # Optimized CTR procedure #
1421 my $key_="r11";
1422 my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
1423 my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14));
1424 my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(15..22));
1425 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
1426 # v26-v31 last 6 round keys
1427 my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment
1428 my ($two,$three,$four)=($outhead,$outperm,$outmask);
1429
1430 $code.=<<___;
1431 .align 5
1432 _aesp8_ctr32_encrypt8x:
1433 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
1434 li r10,`$FRAME+8*16+15`
1435 li r11,`$FRAME+8*16+31`
1436 stvx v20,r10,$sp # ABI says so
1437 addi r10,r10,32
1438 stvx v21,r11,$sp
1439 addi r11,r11,32
1440 stvx v22,r10,$sp
1441 addi r10,r10,32
1442 stvx v23,r11,$sp
1443 addi r11,r11,32
1444 stvx v24,r10,$sp
1445 addi r10,r10,32
1446 stvx v25,r11,$sp
1447 addi r11,r11,32
1448 stvx v26,r10,$sp
1449 addi r10,r10,32
1450 stvx v27,r11,$sp
1451 addi r11,r11,32
1452 stvx v28,r10,$sp
1453 addi r10,r10,32
1454 stvx v29,r11,$sp
1455 addi r11,r11,32
1456 stvx v30,r10,$sp
1457 stvx v31,r11,$sp
1458 li r0,-1
1459 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
1460 li $x10,0x10
1461 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1462 li $x20,0x20
1463 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1464 li $x30,0x30
1465 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1466 li $x40,0x40
1467 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1468 li $x50,0x50
1469 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1470 li $x60,0x60
1471 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1472 li $x70,0x70
1473 mtspr 256,r0
1474
1475 subi $rounds,$rounds,3 # -4 in total
1476
1477 lvx $rndkey0,$x00,$key # load key schedule
1478 lvx v30,$x10,$key
1479 addi $key,$key,0x20
1480 lvx v31,$x00,$key
1481 ?vperm $rndkey0,$rndkey0,v30,$keyperm
1482 addi $key_,$sp,$FRAME+15
1483 mtctr $rounds
1484
1485 Load_ctr32_enc_key:
1486 ?vperm v24,v30,v31,$keyperm
1487 lvx v30,$x10,$key
1488 addi $key,$key,0x20
1489 stvx v24,$x00,$key_ # off-load round[1]
1490 ?vperm v25,v31,v30,$keyperm
1491 lvx v31,$x00,$key
1492 stvx v25,$x10,$key_ # off-load round[2]
1493 addi $key_,$key_,0x20
1494 bdnz Load_ctr32_enc_key
1495
1496 lvx v26,$x10,$key
1497 ?vperm v24,v30,v31,$keyperm
1498 lvx v27,$x20,$key
1499 stvx v24,$x00,$key_ # off-load round[3]
1500 ?vperm v25,v31,v26,$keyperm
1501 lvx v28,$x30,$key
1502 stvx v25,$x10,$key_ # off-load round[4]
1503 addi $key_,$sp,$FRAME+15 # rewind $key_
1504 ?vperm v26,v26,v27,$keyperm
1505 lvx v29,$x40,$key
1506 ?vperm v27,v27,v28,$keyperm
1507 lvx v30,$x50,$key
1508 ?vperm v28,v28,v29,$keyperm
1509 lvx v31,$x60,$key
1510 ?vperm v29,v29,v30,$keyperm
1511 lvx $out0,$x70,$key # borrow $out0
1512 ?vperm v30,v30,v31,$keyperm
1513 lvx v24,$x00,$key_ # pre-load round[1]
1514 ?vperm v31,v31,$out0,$keyperm
1515 lvx v25,$x10,$key_ # pre-load round[2]
1516
1517 vadduqm $two,$one,$one
1518 subi $inp,$inp,15 # undo "caller"
1519 $SHL $len,$len,4
1520
1521 vadduqm $out1,$ivec,$one # counter values ...
1522 vadduqm $out2,$ivec,$two # (do all ctr adds as 128-bit)
1523 vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0]
1524 le?li $idx,8
1525 vadduqm $out3,$out1,$two
1526 vxor $out1,$out1,$rndkey0
1527 le?lvsl $inpperm,0,$idx
1528 vadduqm $out4,$out2,$two
1529 vxor $out2,$out2,$rndkey0
1530 le?vspltisb $tmp,0x0f
1531 vadduqm $out5,$out3,$two
1532 vxor $out3,$out3,$rndkey0
1533 le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u
1534 vadduqm $out6,$out4,$two
1535 vxor $out4,$out4,$rndkey0
1536 vadduqm $out7,$out5,$two
1537 vxor $out5,$out5,$rndkey0
1538 vadduqm $ivec,$out6,$two # next counter value
1539 vxor $out6,$out6,$rndkey0
1540 vxor $out7,$out7,$rndkey0
1541
1542 mtctr $rounds
1543 b Loop_ctr32_enc8x
1544 .align 5
1545 Loop_ctr32_enc8x:
1546 vcipher $out0,$out0,v24
1547 vcipher $out1,$out1,v24
1548 vcipher $out2,$out2,v24
1549 vcipher $out3,$out3,v24
1550 vcipher $out4,$out4,v24
1551 vcipher $out5,$out5,v24
1552 vcipher $out6,$out6,v24
1553 vcipher $out7,$out7,v24
1554 Loop_ctr32_enc8x_middle:
1555 lvx v24,$x20,$key_ # round[3]
1556 addi $key_,$key_,0x20
1557
1558 vcipher $out0,$out0,v25
1559 vcipher $out1,$out1,v25
1560 vcipher $out2,$out2,v25
1561 vcipher $out3,$out3,v25
1562 vcipher $out4,$out4,v25
1563 vcipher $out5,$out5,v25
1564 vcipher $out6,$out6,v25
1565 vcipher $out7,$out7,v25
1566 lvx v25,$x10,$key_ # round[4]
1567 bdnz Loop_ctr32_enc8x
1568
1569 subic r11,$len,256 # $len-256, borrow $key_
1570 vcipher $out0,$out0,v24
1571 vcipher $out1,$out1,v24
1572 vcipher $out2,$out2,v24
1573 vcipher $out3,$out3,v24
1574 vcipher $out4,$out4,v24
1575 vcipher $out5,$out5,v24
1576 vcipher $out6,$out6,v24
1577 vcipher $out7,$out7,v24
1578
1579 subfe r0,r0,r0 # borrow?-1:0
1580 vcipher $out0,$out0,v25
1581 vcipher $out1,$out1,v25
1582 vcipher $out2,$out2,v25
1583 vcipher $out3,$out3,v25
1584 vcipher $out4,$out4,v25
1585 vcipher $out5,$out5,v25
1586 vcipher $out6,$out6,v25
1587 vcipher $out7,$out7,v25
1588
1589 and r0,r0,r11
1590 addi $key_,$sp,$FRAME+15 # rewind $key_
1591 vcipher $out0,$out0,v26
1592 vcipher $out1,$out1,v26
1593 vcipher $out2,$out2,v26
1594 vcipher $out3,$out3,v26
1595 vcipher $out4,$out4,v26
1596 vcipher $out5,$out5,v26
1597 vcipher $out6,$out6,v26
1598 vcipher $out7,$out7,v26
1599 lvx v24,$x00,$key_ # re-pre-load round[1]
1600
1601 subic $len,$len,129 # $len-=129
1602 vcipher $out0,$out0,v27
1603 addi $len,$len,1 # $len-=128 really
1604 vcipher $out1,$out1,v27
1605 vcipher $out2,$out2,v27
1606 vcipher $out3,$out3,v27
1607 vcipher $out4,$out4,v27
1608 vcipher $out5,$out5,v27
1609 vcipher $out6,$out6,v27
1610 vcipher $out7,$out7,v27
1611 lvx v25,$x10,$key_ # re-pre-load round[2]
1612
1613 vcipher $out0,$out0,v28
1614 lvx_u $in0,$x00,$inp # load input
1615 vcipher $out1,$out1,v28
1616 lvx_u $in1,$x10,$inp
1617 vcipher $out2,$out2,v28
1618 lvx_u $in2,$x20,$inp
1619 vcipher $out3,$out3,v28
1620 lvx_u $in3,$x30,$inp
1621 vcipher $out4,$out4,v28
1622 lvx_u $in4,$x40,$inp
1623 vcipher $out5,$out5,v28
1624 lvx_u $in5,$x50,$inp
1625 vcipher $out6,$out6,v28
1626 lvx_u $in6,$x60,$inp
1627 vcipher $out7,$out7,v28
1628 lvx_u $in7,$x70,$inp
1629 addi $inp,$inp,0x80
1630
1631 vcipher $out0,$out0,v29
1632 le?vperm $in0,$in0,$in0,$inpperm
1633 vcipher $out1,$out1,v29
1634 le?vperm $in1,$in1,$in1,$inpperm
1635 vcipher $out2,$out2,v29
1636 le?vperm $in2,$in2,$in2,$inpperm
1637 vcipher $out3,$out3,v29
1638 le?vperm $in3,$in3,$in3,$inpperm
1639 vcipher $out4,$out4,v29
1640 le?vperm $in4,$in4,$in4,$inpperm
1641 vcipher $out5,$out5,v29
1642 le?vperm $in5,$in5,$in5,$inpperm
1643 vcipher $out6,$out6,v29
1644 le?vperm $in6,$in6,$in6,$inpperm
1645 vcipher $out7,$out7,v29
1646 le?vperm $in7,$in7,$in7,$inpperm
1647
1648 add $inp,$inp,r0 # $inp is adjusted in such
1649 # way that at exit from the
1650 # loop inX-in7 are loaded
1651 # with last "words"
1652 subfe. r0,r0,r0 # borrow?-1:0
1653 vcipher $out0,$out0,v30
1654 vxor $in0,$in0,v31 # xor with last round key
1655 vcipher $out1,$out1,v30
1656 vxor $in1,$in1,v31
1657 vcipher $out2,$out2,v30
1658 vxor $in2,$in2,v31
1659 vcipher $out3,$out3,v30
1660 vxor $in3,$in3,v31
1661 vcipher $out4,$out4,v30
1662 vxor $in4,$in4,v31
1663 vcipher $out5,$out5,v30
1664 vxor $in5,$in5,v31
1665 vcipher $out6,$out6,v30
1666 vxor $in6,$in6,v31
1667 vcipher $out7,$out7,v30
1668 vxor $in7,$in7,v31
1669
1670 bne Lctr32_enc8x_break # did $len-129 borrow?
1671
1672 vcipherlast $in0,$out0,$in0
1673 vcipherlast $in1,$out1,$in1
1674 vadduqm $out1,$ivec,$one # counter values ...
1675 vcipherlast $in2,$out2,$in2
1676 vadduqm $out2,$ivec,$two
1677 vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0]
1678 vcipherlast $in3,$out3,$in3
1679 vadduqm $out3,$out1,$two
1680 vxor $out1,$out1,$rndkey0
1681 vcipherlast $in4,$out4,$in4
1682 vadduqm $out4,$out2,$two
1683 vxor $out2,$out2,$rndkey0
1684 vcipherlast $in5,$out5,$in5
1685 vadduqm $out5,$out3,$two
1686 vxor $out3,$out3,$rndkey0
1687 vcipherlast $in6,$out6,$in6
1688 vadduqm $out6,$out4,$two
1689 vxor $out4,$out4,$rndkey0
1690 vcipherlast $in7,$out7,$in7
1691 vadduqm $out7,$out5,$two
1692 vxor $out5,$out5,$rndkey0
1693 le?vperm $in0,$in0,$in0,$inpperm
1694 vadduqm $ivec,$out6,$two # next counter value
1695 vxor $out6,$out6,$rndkey0
1696 le?vperm $in1,$in1,$in1,$inpperm
1697 vxor $out7,$out7,$rndkey0
1698 mtctr $rounds
1699
1700 vcipher $out0,$out0,v24
1701 stvx_u $in0,$x00,$out
1702 le?vperm $in2,$in2,$in2,$inpperm
1703 vcipher $out1,$out1,v24
1704 stvx_u $in1,$x10,$out
1705 le?vperm $in3,$in3,$in3,$inpperm
1706 vcipher $out2,$out2,v24
1707 stvx_u $in2,$x20,$out
1708 le?vperm $in4,$in4,$in4,$inpperm
1709 vcipher $out3,$out3,v24
1710 stvx_u $in3,$x30,$out
1711 le?vperm $in5,$in5,$in5,$inpperm
1712 vcipher $out4,$out4,v24
1713 stvx_u $in4,$x40,$out
1714 le?vperm $in6,$in6,$in6,$inpperm
1715 vcipher $out5,$out5,v24
1716 stvx_u $in5,$x50,$out
1717 le?vperm $in7,$in7,$in7,$inpperm
1718 vcipher $out6,$out6,v24
1719 stvx_u $in6,$x60,$out
1720 vcipher $out7,$out7,v24
1721 stvx_u $in7,$x70,$out
1722 addi $out,$out,0x80
1723
1724 b Loop_ctr32_enc8x_middle
1725
1726 .align 5
1727 Lctr32_enc8x_break:
1728 cmpwi $len,-0x60
1729 blt Lctr32_enc8x_one
1730 nop
1731 beq Lctr32_enc8x_two
1732 cmpwi $len,-0x40
1733 blt Lctr32_enc8x_three
1734 nop
1735 beq Lctr32_enc8x_four
1736 cmpwi $len,-0x20
1737 blt Lctr32_enc8x_five
1738 nop
1739 beq Lctr32_enc8x_six
1740 cmpwi $len,0x00
1741 blt Lctr32_enc8x_seven
1742
1743 Lctr32_enc8x_eight:
1744 vcipherlast $out0,$out0,$in0
1745 vcipherlast $out1,$out1,$in1
1746 vcipherlast $out2,$out2,$in2
1747 vcipherlast $out3,$out3,$in3
1748 vcipherlast $out4,$out4,$in4
1749 vcipherlast $out5,$out5,$in5
1750 vcipherlast $out6,$out6,$in6
1751 vcipherlast $out7,$out7,$in7
1752
1753 le?vperm $out0,$out0,$out0,$inpperm
1754 le?vperm $out1,$out1,$out1,$inpperm
1755 stvx_u $out0,$x00,$out
1756 le?vperm $out2,$out2,$out2,$inpperm
1757 stvx_u $out1,$x10,$out
1758 le?vperm $out3,$out3,$out3,$inpperm
1759 stvx_u $out2,$x20,$out
1760 le?vperm $out4,$out4,$out4,$inpperm
1761 stvx_u $out3,$x30,$out
1762 le?vperm $out5,$out5,$out5,$inpperm
1763 stvx_u $out4,$x40,$out
1764 le?vperm $out6,$out6,$out6,$inpperm
1765 stvx_u $out5,$x50,$out
1766 le?vperm $out7,$out7,$out7,$inpperm
1767 stvx_u $out6,$x60,$out
1768 stvx_u $out7,$x70,$out
1769 addi $out,$out,0x80
1770 b Lctr32_enc8x_done
1771
1772 .align 5
1773 Lctr32_enc8x_seven:
1774 vcipherlast $out0,$out0,$in1
1775 vcipherlast $out1,$out1,$in2
1776 vcipherlast $out2,$out2,$in3
1777 vcipherlast $out3,$out3,$in4
1778 vcipherlast $out4,$out4,$in5
1779 vcipherlast $out5,$out5,$in6
1780 vcipherlast $out6,$out6,$in7
1781
1782 le?vperm $out0,$out0,$out0,$inpperm
1783 le?vperm $out1,$out1,$out1,$inpperm
1784 stvx_u $out0,$x00,$out
1785 le?vperm $out2,$out2,$out2,$inpperm
1786 stvx_u $out1,$x10,$out
1787 le?vperm $out3,$out3,$out3,$inpperm
1788 stvx_u $out2,$x20,$out
1789 le?vperm $out4,$out4,$out4,$inpperm
1790 stvx_u $out3,$x30,$out
1791 le?vperm $out5,$out5,$out5,$inpperm
1792 stvx_u $out4,$x40,$out
1793 le?vperm $out6,$out6,$out6,$inpperm
1794 stvx_u $out5,$x50,$out
1795 stvx_u $out6,$x60,$out
1796 addi $out,$out,0x70
1797 b Lctr32_enc8x_done
1798
1799 .align 5
1800 Lctr32_enc8x_six:
1801 vcipherlast $out0,$out0,$in2
1802 vcipherlast $out1,$out1,$in3
1803 vcipherlast $out2,$out2,$in4
1804 vcipherlast $out3,$out3,$in5
1805 vcipherlast $out4,$out4,$in6
1806 vcipherlast $out5,$out5,$in7
1807
1808 le?vperm $out0,$out0,$out0,$inpperm
1809 le?vperm $out1,$out1,$out1,$inpperm
1810 stvx_u $out0,$x00,$out
1811 le?vperm $out2,$out2,$out2,$inpperm
1812 stvx_u $out1,$x10,$out
1813 le?vperm $out3,$out3,$out3,$inpperm
1814 stvx_u $out2,$x20,$out
1815 le?vperm $out4,$out4,$out4,$inpperm
1816 stvx_u $out3,$x30,$out
1817 le?vperm $out5,$out5,$out5,$inpperm
1818 stvx_u $out4,$x40,$out
1819 stvx_u $out5,$x50,$out
1820 addi $out,$out,0x60
1821 b Lctr32_enc8x_done
1822
1823 .align 5
1824 Lctr32_enc8x_five:
1825 vcipherlast $out0,$out0,$in3
1826 vcipherlast $out1,$out1,$in4
1827 vcipherlast $out2,$out2,$in5
1828 vcipherlast $out3,$out3,$in6
1829 vcipherlast $out4,$out4,$in7
1830
1831 le?vperm $out0,$out0,$out0,$inpperm
1832 le?vperm $out1,$out1,$out1,$inpperm
1833 stvx_u $out0,$x00,$out
1834 le?vperm $out2,$out2,$out2,$inpperm
1835 stvx_u $out1,$x10,$out
1836 le?vperm $out3,$out3,$out3,$inpperm
1837 stvx_u $out2,$x20,$out
1838 le?vperm $out4,$out4,$out4,$inpperm
1839 stvx_u $out3,$x30,$out
1840 stvx_u $out4,$x40,$out
1841 addi $out,$out,0x50
1842 b Lctr32_enc8x_done
1843
1844 .align 5
1845 Lctr32_enc8x_four:
1846 vcipherlast $out0,$out0,$in4
1847 vcipherlast $out1,$out1,$in5
1848 vcipherlast $out2,$out2,$in6
1849 vcipherlast $out3,$out3,$in7
1850
1851 le?vperm $out0,$out0,$out0,$inpperm
1852 le?vperm $out1,$out1,$out1,$inpperm
1853 stvx_u $out0,$x00,$out
1854 le?vperm $out2,$out2,$out2,$inpperm
1855 stvx_u $out1,$x10,$out
1856 le?vperm $out3,$out3,$out3,$inpperm
1857 stvx_u $out2,$x20,$out
1858 stvx_u $out3,$x30,$out
1859 addi $out,$out,0x40
1860 b Lctr32_enc8x_done
1861
1862 .align 5
1863 Lctr32_enc8x_three:
1864 vcipherlast $out0,$out0,$in5
1865 vcipherlast $out1,$out1,$in6
1866 vcipherlast $out2,$out2,$in7
1867
1868 le?vperm $out0,$out0,$out0,$inpperm
1869 le?vperm $out1,$out1,$out1,$inpperm
1870 stvx_u $out0,$x00,$out
1871 le?vperm $out2,$out2,$out2,$inpperm
1872 stvx_u $out1,$x10,$out
1873 stvx_u $out2,$x20,$out
1874 addi $out,$out,0x30
1875 b Lctr32_enc8x_done
1876
1877 .align 5
1878 Lctr32_enc8x_two:
1879 vcipherlast $out0,$out0,$in6
1880 vcipherlast $out1,$out1,$in7
1881
1882 le?vperm $out0,$out0,$out0,$inpperm
1883 le?vperm $out1,$out1,$out1,$inpperm
1884 stvx_u $out0,$x00,$out
1885 stvx_u $out1,$x10,$out
1886 addi $out,$out,0x20
1887 b Lctr32_enc8x_done
1888
1889 .align 5
1890 Lctr32_enc8x_one:
1891 vcipherlast $out0,$out0,$in7
1892
1893 le?vperm $out0,$out0,$out0,$inpperm
1894 stvx_u $out0,0,$out
1895 addi $out,$out,0x10
1896
1897 Lctr32_enc8x_done:
1898 li r10,`$FRAME+15`
1899 li r11,`$FRAME+31`
1900 stvx $inpperm,r10,$sp # wipe copies of round keys
1901 addi r10,r10,32
1902 stvx $inpperm,r11,$sp
1903 addi r11,r11,32
1904 stvx $inpperm,r10,$sp
1905 addi r10,r10,32
1906 stvx $inpperm,r11,$sp
1907 addi r11,r11,32
1908 stvx $inpperm,r10,$sp
1909 addi r10,r10,32
1910 stvx $inpperm,r11,$sp
1911 addi r11,r11,32
1912 stvx $inpperm,r10,$sp
1913 addi r10,r10,32
1914 stvx $inpperm,r11,$sp
1915 addi r11,r11,32
1916
1917 mtspr 256,$vrsave
1918 lvx v20,r10,$sp # ABI says so
1919 addi r10,r10,32
1920 lvx v21,r11,$sp
1921 addi r11,r11,32
1922 lvx v22,r10,$sp
1923 addi r10,r10,32
1924 lvx v23,r11,$sp
1925 addi r11,r11,32
1926 lvx v24,r10,$sp
1927 addi r10,r10,32
1928 lvx v25,r11,$sp
1929 addi r11,r11,32
1930 lvx v26,r10,$sp
1931 addi r10,r10,32
1932 lvx v27,r11,$sp
1933 addi r11,r11,32
1934 lvx v28,r10,$sp
1935 addi r10,r10,32
1936 lvx v29,r11,$sp
1937 addi r11,r11,32
1938 lvx v30,r10,$sp
1939 lvx v31,r11,$sp
1940 $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1941 $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1942 $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1943 $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1944 $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1945 $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1946 addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
1947 blr
1948 .long 0
1949 .byte 0,12,0x14,0,0x80,6,6,0
1950 .long 0
1951 .size .${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks
1952 ___
1953 }} }}}
1954
1955 #########################################################################
1956 {{{ # XTS procedures #
1957 # int aes_p8_xts_[en|de]crypt(const char *inp, char *out, size_t len, #
1958 # const AES_KEY *key1, const AES_KEY *key2, #
1959 # [const] unsigned char iv[16]); #
1960 # If $key2 is NULL, then a "tweak chaining" mode is engaged, in which #
1961 # input tweak value is assumed to be encrypted already, and last tweak #
1962 # value, one suitable for consecutive call on same chunk of data, is #
1963 # written back to original buffer. In addition, in "tweak chaining" #
1964 # mode only complete input blocks are processed. #
1965
1966 my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) = map("r$_",(3..10));
1967 my ($rndkey0,$rndkey1,$inout) = map("v$_",(0..2));
1968 my ($output,$inptail,$inpperm,$leperm,$keyperm) = map("v$_",(3..7));
1969 my ($tweak,$seven,$eighty7,$tmp,$tweak1) = map("v$_",(8..12));
1970 my $taillen = $key2;
1971
1972 ($inp,$idx) = ($idx,$inp); # reassign
1973
1974 $code.=<<___;
1975 .globl .${prefix}_xts_encrypt
1976 mr $inp,r3 # reassign
1977 li r3,-1
1978 ${UCMP}i $len,16
1979 bltlr-
1980
1981 lis r0,0xfff0
1982 mfspr r12,256 # save vrsave
1983 li r11,0
1984 mtspr 256,r0
1985
1986 vspltisb $seven,0x07 # 0x070707..07
1987 le?lvsl $leperm,r11,r11
1988 le?vspltisb $tmp,0x0f
1989 le?vxor $leperm,$leperm,$seven
1990
1991 li $idx,15
1992 lvx $tweak,0,$ivp # load [unaligned] iv
1993 lvsl $inpperm,0,$ivp
1994 lvx $inptail,$idx,$ivp
1995 le?vxor $inpperm,$inpperm,$tmp
1996 vperm $tweak,$tweak,$inptail,$inpperm
1997
1998 neg r11,$inp
1999 lvsr $inpperm,0,r11 # prepare for unaligned load
2000 lvx $inout,0,$inp
2001 addi $inp,$inp,15 # 15 is not typo
2002 le?vxor $inpperm,$inpperm,$tmp
2003
2004 ${UCMP}i $key2,0 # key2==NULL?
2005 beq Lxts_enc_no_key2
2006
2007 ?lvsl $keyperm,0,$key2 # prepare for unaligned key
2008 lwz $rounds,240($key2)
2009 srwi $rounds,$rounds,1
2010 subi $rounds,$rounds,1
2011 li $idx,16
2012
2013 lvx $rndkey0,0,$key2
2014 lvx $rndkey1,$idx,$key2
2015 addi $idx,$idx,16
2016 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2017 vxor $tweak,$tweak,$rndkey0
2018 lvx $rndkey0,$idx,$key2
2019 addi $idx,$idx,16
2020 mtctr $rounds
2021
2022 Ltweak_xts_enc:
2023 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2024 vcipher $tweak,$tweak,$rndkey1
2025 lvx $rndkey1,$idx,$key2
2026 addi $idx,$idx,16
2027 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2028 vcipher $tweak,$tweak,$rndkey0
2029 lvx $rndkey0,$idx,$key2
2030 addi $idx,$idx,16
2031 bdnz Ltweak_xts_enc
2032
2033 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2034 vcipher $tweak,$tweak,$rndkey1
2035 lvx $rndkey1,$idx,$key2
2036 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2037 vcipherlast $tweak,$tweak,$rndkey0
2038
2039 li $ivp,0 # don't chain the tweak
2040 b Lxts_enc
2041
2042 Lxts_enc_no_key2:
2043 li $idx,-16
2044 and $len,$len,$idx # in "tweak chaining"
2045 # mode only complete
2046 # blocks are processed
2047 Lxts_enc:
2048 lvx $inptail,0,$inp
2049 addi $inp,$inp,16
2050
2051 ?lvsl $keyperm,0,$key1 # prepare for unaligned key
2052 lwz $rounds,240($key1)
2053 srwi $rounds,$rounds,1
2054 subi $rounds,$rounds,1
2055 li $idx,16
2056
2057 vslb $eighty7,$seven,$seven # 0x808080..80
2058 vor $eighty7,$eighty7,$seven # 0x878787..87
2059 vspltisb $tmp,1 # 0x010101..01
2060 vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01
2061
2062 ${UCMP}i $len,96
2063 bge _aesp8_xts_encrypt6x
2064
2065 andi. $taillen,$len,15
2066 subic r0,$len,32
2067 subi $taillen,$taillen,16
2068 subfe r0,r0,r0
2069 and r0,r0,$taillen
2070 add $inp,$inp,r0
2071
2072 lvx $rndkey0,0,$key1
2073 lvx $rndkey1,$idx,$key1
2074 addi $idx,$idx,16
2075 vperm $inout,$inout,$inptail,$inpperm
2076 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2077 vxor $inout,$inout,$tweak
2078 vxor $inout,$inout,$rndkey0
2079 lvx $rndkey0,$idx,$key1
2080 addi $idx,$idx,16
2081 mtctr $rounds
2082 b Loop_xts_enc
2083
2084 .align 5
2085 Loop_xts_enc:
2086 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2087 vcipher $inout,$inout,$rndkey1
2088 lvx $rndkey1,$idx,$key1
2089 addi $idx,$idx,16
2090 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2091 vcipher $inout,$inout,$rndkey0
2092 lvx $rndkey0,$idx,$key1
2093 addi $idx,$idx,16
2094 bdnz Loop_xts_enc
2095
2096 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2097 vcipher $inout,$inout,$rndkey1
2098 lvx $rndkey1,$idx,$key1
2099 li $idx,16
2100 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2101 vxor $rndkey0,$rndkey0,$tweak
2102 vcipherlast $output,$inout,$rndkey0
2103
2104 le?vperm $tmp,$output,$output,$leperm
2105 be?nop
2106 le?stvx_u $tmp,0,$out
2107 be?stvx_u $output,0,$out
2108 addi $out,$out,16
2109
2110 subic. $len,$len,16
2111 beq Lxts_enc_done
2112
2113 vmr $inout,$inptail
2114 lvx $inptail,0,$inp
2115 addi $inp,$inp,16
2116 lvx $rndkey0,0,$key1
2117 lvx $rndkey1,$idx,$key1
2118 addi $idx,$idx,16
2119
2120 subic r0,$len,32
2121 subfe r0,r0,r0
2122 and r0,r0,$taillen
2123 add $inp,$inp,r0
2124
2125 vsrab $tmp,$tweak,$seven # next tweak value
2126 vaddubm $tweak,$tweak,$tweak
2127 vsldoi $tmp,$tmp,$tmp,15
2128 vand $tmp,$tmp,$eighty7
2129 vxor $tweak,$tweak,$tmp
2130
2131 vperm $inout,$inout,$inptail,$inpperm
2132 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2133 vxor $inout,$inout,$tweak
2134 vxor $output,$output,$rndkey0 # just in case $len<16
2135 vxor $inout,$inout,$rndkey0
2136 lvx $rndkey0,$idx,$key1
2137 addi $idx,$idx,16
2138
2139 mtctr $rounds
2140 ${UCMP}i $len,16
2141 bge Loop_xts_enc
2142
2143 vxor $output,$output,$tweak
2144 lvsr $inpperm,0,$len # $inpperm is no longer needed
2145 vxor $inptail,$inptail,$inptail # $inptail is no longer needed
2146 vspltisb $tmp,-1
2147 vperm $inptail,$inptail,$tmp,$inpperm
2148 vsel $inout,$inout,$output,$inptail
2149
2150 subi r11,$out,17
2151 subi $out,$out,16
2152 mtctr $len
2153 li $len,16
2154 Loop_xts_enc_steal:
2155 lbzu r0,1(r11)
2156 stb r0,16(r11)
2157 bdnz Loop_xts_enc_steal
2158
2159 mtctr $rounds
2160 b Loop_xts_enc # one more time...
2161
2162 Lxts_enc_done:
2163 ${UCMP}i $ivp,0
2164 beq Lxts_enc_ret
2165
2166 vsrab $tmp,$tweak,$seven # next tweak value
2167 vaddubm $tweak,$tweak,$tweak
2168 vsldoi $tmp,$tmp,$tmp,15
2169 vand $tmp,$tmp,$eighty7
2170 vxor $tweak,$tweak,$tmp
2171
2172 le?vperm $tweak,$tweak,$tweak,$leperm
2173 stvx_u $tweak,0,$ivp
2174
2175 Lxts_enc_ret:
2176 mtspr 256,r12 # restore vrsave
2177 li r3,0
2178 blr
2179 .long 0
2180 .byte 0,12,0x04,0,0x80,6,6,0
2181 .long 0
2182 .size .${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt
2183
2184 .globl .${prefix}_xts_decrypt
2185 mr $inp,r3 # reassign
2186 li r3,-1
2187 ${UCMP}i $len,16
2188 bltlr-
2189
2190 lis r0,0xfff8
2191 mfspr r12,256 # save vrsave
2192 li r11,0
2193 mtspr 256,r0
2194
2195 andi. r0,$len,15
2196 neg r0,r0
2197 andi. r0,r0,16
2198 sub $len,$len,r0
2199
2200 vspltisb $seven,0x07 # 0x070707..07
2201 le?lvsl $leperm,r11,r11
2202 le?vspltisb $tmp,0x0f
2203 le?vxor $leperm,$leperm,$seven
2204
2205 li $idx,15
2206 lvx $tweak,0,$ivp # load [unaligned] iv
2207 lvsl $inpperm,0,$ivp
2208 lvx $inptail,$idx,$ivp
2209 le?vxor $inpperm,$inpperm,$tmp
2210 vperm $tweak,$tweak,$inptail,$inpperm
2211
2212 neg r11,$inp
2213 lvsr $inpperm,0,r11 # prepare for unaligned load
2214 lvx $inout,0,$inp
2215 addi $inp,$inp,15 # 15 is not typo
2216 le?vxor $inpperm,$inpperm,$tmp
2217
2218 ${UCMP}i $key2,0 # key2==NULL?
2219 beq Lxts_dec_no_key2
2220
2221 ?lvsl $keyperm,0,$key2 # prepare for unaligned key
2222 lwz $rounds,240($key2)
2223 srwi $rounds,$rounds,1
2224 subi $rounds,$rounds,1
2225 li $idx,16
2226
2227 lvx $rndkey0,0,$key2
2228 lvx $rndkey1,$idx,$key2
2229 addi $idx,$idx,16
2230 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2231 vxor $tweak,$tweak,$rndkey0
2232 lvx $rndkey0,$idx,$key2
2233 addi $idx,$idx,16
2234 mtctr $rounds
2235
2236 Ltweak_xts_dec:
2237 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2238 vcipher $tweak,$tweak,$rndkey1
2239 lvx $rndkey1,$idx,$key2
2240 addi $idx,$idx,16
2241 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2242 vcipher $tweak,$tweak,$rndkey0
2243 lvx $rndkey0,$idx,$key2
2244 addi $idx,$idx,16
2245 bdnz Ltweak_xts_dec
2246
2247 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2248 vcipher $tweak,$tweak,$rndkey1
2249 lvx $rndkey1,$idx,$key2
2250 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2251 vcipherlast $tweak,$tweak,$rndkey0
2252
2253 li $ivp,0 # don't chain the tweak
2254 b Lxts_dec
2255
2256 Lxts_dec_no_key2:
2257 neg $idx,$len
2258 andi. $idx,$idx,15
2259 add $len,$len,$idx # in "tweak chaining"
2260 # mode only complete
2261 # blocks are processed
2262 Lxts_dec:
2263 lvx $inptail,0,$inp
2264 addi $inp,$inp,16
2265
2266 ?lvsl $keyperm,0,$key1 # prepare for unaligned key
2267 lwz $rounds,240($key1)
2268 srwi $rounds,$rounds,1
2269 subi $rounds,$rounds,1
2270 li $idx,16
2271
2272 vslb $eighty7,$seven,$seven # 0x808080..80
2273 vor $eighty7,$eighty7,$seven # 0x878787..87
2274 vspltisb $tmp,1 # 0x010101..01
2275 vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01
2276
2277 ${UCMP}i $len,96
2278 bge _aesp8_xts_decrypt6x
2279
2280 lvx $rndkey0,0,$key1
2281 lvx $rndkey1,$idx,$key1
2282 addi $idx,$idx,16
2283 vperm $inout,$inout,$inptail,$inpperm
2284 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2285 vxor $inout,$inout,$tweak
2286 vxor $inout,$inout,$rndkey0
2287 lvx $rndkey0,$idx,$key1
2288 addi $idx,$idx,16
2289 mtctr $rounds
2290
2291 ${UCMP}i $len,16
2292 blt Ltail_xts_dec
2293 be?b Loop_xts_dec
2294
2295 .align 5
2296 Loop_xts_dec:
2297 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2298 vncipher $inout,$inout,$rndkey1
2299 lvx $rndkey1,$idx,$key1
2300 addi $idx,$idx,16
2301 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2302 vncipher $inout,$inout,$rndkey0
2303 lvx $rndkey0,$idx,$key1
2304 addi $idx,$idx,16
2305 bdnz Loop_xts_dec
2306
2307 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2308 vncipher $inout,$inout,$rndkey1
2309 lvx $rndkey1,$idx,$key1
2310 li $idx,16
2311 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2312 vxor $rndkey0,$rndkey0,$tweak
2313 vncipherlast $output,$inout,$rndkey0
2314
2315 le?vperm $tmp,$output,$output,$leperm
2316 be?nop
2317 le?stvx_u $tmp,0,$out
2318 be?stvx_u $output,0,$out
2319 addi $out,$out,16
2320
2321 subic. $len,$len,16
2322 beq Lxts_dec_done
2323
2324 vmr $inout,$inptail
2325 lvx $inptail,0,$inp
2326 addi $inp,$inp,16
2327 lvx $rndkey0,0,$key1
2328 lvx $rndkey1,$idx,$key1
2329 addi $idx,$idx,16
2330
2331 vsrab $tmp,$tweak,$seven # next tweak value
2332 vaddubm $tweak,$tweak,$tweak
2333 vsldoi $tmp,$tmp,$tmp,15
2334 vand $tmp,$tmp,$eighty7
2335 vxor $tweak,$tweak,$tmp
2336
2337 vperm $inout,$inout,$inptail,$inpperm
2338 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2339 vxor $inout,$inout,$tweak
2340 vxor $inout,$inout,$rndkey0
2341 lvx $rndkey0,$idx,$key1
2342 addi $idx,$idx,16
2343
2344 mtctr $rounds
2345 ${UCMP}i $len,16
2346 bge Loop_xts_dec
2347
2348 Ltail_xts_dec:
2349 vsrab $tmp,$tweak,$seven # next tweak value
2350 vaddubm $tweak1,$tweak,$tweak
2351 vsldoi $tmp,$tmp,$tmp,15
2352 vand $tmp,$tmp,$eighty7
2353 vxor $tweak1,$tweak1,$tmp
2354
2355 subi $inp,$inp,16
2356 add $inp,$inp,$len
2357
2358 vxor $inout,$inout,$tweak # :-(
2359 vxor $inout,$inout,$tweak1 # :-)
2360
2361 Loop_xts_dec_short:
2362 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2363 vncipher $inout,$inout,$rndkey1
2364 lvx $rndkey1,$idx,$key1
2365 addi $idx,$idx,16
2366 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2367 vncipher $inout,$inout,$rndkey0
2368 lvx $rndkey0,$idx,$key1
2369 addi $idx,$idx,16
2370 bdnz Loop_xts_dec_short
2371
2372 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2373 vncipher $inout,$inout,$rndkey1
2374 lvx $rndkey1,$idx,$key1
2375 li $idx,16
2376 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2377 vxor $rndkey0,$rndkey0,$tweak1
2378 vncipherlast $output,$inout,$rndkey0
2379
2380 le?vperm $tmp,$output,$output,$leperm
2381 be?nop
2382 le?stvx_u $tmp,0,$out
2383 be?stvx_u $output,0,$out
2384
2385 vmr $inout,$inptail
2386 lvx $inptail,0,$inp
2387 #addi $inp,$inp,16
2388 lvx $rndkey0,0,$key1
2389 lvx $rndkey1,$idx,$key1
2390 addi $idx,$idx,16
2391 vperm $inout,$inout,$inptail,$inpperm
2392 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2393
2394 lvsr $inpperm,0,$len # $inpperm is no longer needed
2395 vxor $inptail,$inptail,$inptail # $inptail is no longer needed
2396 vspltisb $tmp,-1
2397 vperm $inptail,$inptail,$tmp,$inpperm
2398 vsel $inout,$inout,$output,$inptail
2399
2400 vxor $rndkey0,$rndkey0,$tweak
2401 vxor $inout,$inout,$rndkey0
2402 lvx $rndkey0,$idx,$key1
2403 addi $idx,$idx,16
2404
2405 subi r11,$out,1
2406 mtctr $len
2407 li $len,16
2408 Loop_xts_dec_steal:
2409 lbzu r0,1(r11)
2410 stb r0,16(r11)
2411 bdnz Loop_xts_dec_steal
2412
2413 mtctr $rounds
2414 b Loop_xts_dec # one more time...
2415
2416 Lxts_dec_done:
2417 ${UCMP}i $ivp,0
2418 beq Lxts_dec_ret
2419
2420 vsrab $tmp,$tweak,$seven # next tweak value
2421 vaddubm $tweak,$tweak,$tweak
2422 vsldoi $tmp,$tmp,$tmp,15
2423 vand $tmp,$tmp,$eighty7
2424 vxor $tweak,$tweak,$tmp
2425
2426 le?vperm $tweak,$tweak,$tweak,$leperm
2427 stvx_u $tweak,0,$ivp
2428
2429 Lxts_dec_ret:
2430 mtspr 256,r12 # restore vrsave
2431 li r3,0
2432 blr
2433 .long 0
2434 .byte 0,12,0x04,0,0x80,6,6,0
2435 .long 0
2436 .size .${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt
2437 ___
2438 #########################################################################
2439 {{ # Optimized XTS procedures #
2440 my $key_=$key2;
2441 my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31));
2442 $x00=0 if ($flavour =~ /osx/);
2443 my ($in0, $in1, $in2, $in3, $in4, $in5 )=map("v$_",(0..5));
2444 my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16));
2445 my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22));
2446 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
2447 # v26-v31 last 6 round keys
2448 my ($keyperm)=($out0); # aliases with "caller", redundant assignment
2449 my $taillen=$x70;
2450
2451 $code.=<<___;
2452 .align 5
2453 _aesp8_xts_encrypt6x:
2454 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
2455 mflr r11
2456 li r7,`$FRAME+8*16+15`
2457 li r3,`$FRAME+8*16+31`
2458 $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
2459 stvx v20,r7,$sp # ABI says so
2460 addi r7,r7,32
2461 stvx v21,r3,$sp
2462 addi r3,r3,32
2463 stvx v22,r7,$sp
2464 addi r7,r7,32
2465 stvx v23,r3,$sp
2466 addi r3,r3,32
2467 stvx v24,r7,$sp
2468 addi r7,r7,32
2469 stvx v25,r3,$sp
2470 addi r3,r3,32
2471 stvx v26,r7,$sp
2472 addi r7,r7,32
2473 stvx v27,r3,$sp
2474 addi r3,r3,32
2475 stvx v28,r7,$sp
2476 addi r7,r7,32
2477 stvx v29,r3,$sp
2478 addi r3,r3,32
2479 stvx v30,r7,$sp
2480 stvx v31,r3,$sp
2481 li r0,-1
2482 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
2483 li $x10,0x10
2484 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
2485 li $x20,0x20
2486 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
2487 li $x30,0x30
2488 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
2489 li $x40,0x40
2490 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
2491 li $x50,0x50
2492 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
2493 li $x60,0x60
2494 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
2495 li $x70,0x70
2496 mtspr 256,r0
2497
2498 subi $rounds,$rounds,3 # -4 in total
2499
2500 lvx $rndkey0,$x00,$key1 # load key schedule
2501 lvx v30,$x10,$key1
2502 addi $key1,$key1,0x20
2503 lvx v31,$x00,$key1
2504 ?vperm $rndkey0,$rndkey0,v30,$keyperm
2505 addi $key_,$sp,$FRAME+15
2506 mtctr $rounds
2507
2508 Load_xts_enc_key:
2509 ?vperm v24,v30,v31,$keyperm
2510 lvx v30,$x10,$key1
2511 addi $key1,$key1,0x20
2512 stvx v24,$x00,$key_ # off-load round[1]
2513 ?vperm v25,v31,v30,$keyperm
2514 lvx v31,$x00,$key1
2515 stvx v25,$x10,$key_ # off-load round[2]
2516 addi $key_,$key_,0x20
2517 bdnz Load_xts_enc_key
2518
2519 lvx v26,$x10,$key1
2520 ?vperm v24,v30,v31,$keyperm
2521 lvx v27,$x20,$key1
2522 stvx v24,$x00,$key_ # off-load round[3]
2523 ?vperm v25,v31,v26,$keyperm
2524 lvx v28,$x30,$key1
2525 stvx v25,$x10,$key_ # off-load round[4]
2526 addi $key_,$sp,$FRAME+15 # rewind $key_
2527 ?vperm v26,v26,v27,$keyperm
2528 lvx v29,$x40,$key1
2529 ?vperm v27,v27,v28,$keyperm
2530 lvx v30,$x50,$key1
2531 ?vperm v28,v28,v29,$keyperm
2532 lvx v31,$x60,$key1
2533 ?vperm v29,v29,v30,$keyperm
2534 lvx $twk5,$x70,$key1 # borrow $twk5
2535 ?vperm v30,v30,v31,$keyperm
2536 lvx v24,$x00,$key_ # pre-load round[1]
2537 ?vperm v31,v31,$twk5,$keyperm
2538 lvx v25,$x10,$key_ # pre-load round[2]
2539
2540 vperm $in0,$inout,$inptail,$inpperm
2541 subi $inp,$inp,31 # undo "caller"
2542 vxor $twk0,$tweak,$rndkey0
2543 vsrab $tmp,$tweak,$seven # next tweak value
2544 vaddubm $tweak,$tweak,$tweak
2545 vsldoi $tmp,$tmp,$tmp,15
2546 vand $tmp,$tmp,$eighty7
2547 vxor $out0,$in0,$twk0
2548 vxor $tweak,$tweak,$tmp
2549
2550 lvx_u $in1,$x10,$inp
2551 vxor $twk1,$tweak,$rndkey0
2552 vsrab $tmp,$tweak,$seven # next tweak value
2553 vaddubm $tweak,$tweak,$tweak
2554 vsldoi $tmp,$tmp,$tmp,15
2555 le?vperm $in1,$in1,$in1,$leperm
2556 vand $tmp,$tmp,$eighty7
2557 vxor $out1,$in1,$twk1
2558 vxor $tweak,$tweak,$tmp
2559
2560 lvx_u $in2,$x20,$inp
2561 andi. $taillen,$len,15
2562 vxor $twk2,$tweak,$rndkey0
2563 vsrab $tmp,$tweak,$seven # next tweak value
2564 vaddubm $tweak,$tweak,$tweak
2565 vsldoi $tmp,$tmp,$tmp,15
2566 le?vperm $in2,$in2,$in2,$leperm
2567 vand $tmp,$tmp,$eighty7
2568 vxor $out2,$in2,$twk2
2569 vxor $tweak,$tweak,$tmp
2570
2571 lvx_u $in3,$x30,$inp
2572 sub $len,$len,$taillen
2573 vxor $twk3,$tweak,$rndkey0
2574 vsrab $tmp,$tweak,$seven # next tweak value
2575 vaddubm $tweak,$tweak,$tweak
2576 vsldoi $tmp,$tmp,$tmp,15
2577 le?vperm $in3,$in3,$in3,$leperm
2578 vand $tmp,$tmp,$eighty7
2579 vxor $out3,$in3,$twk3
2580 vxor $tweak,$tweak,$tmp
2581
2582 lvx_u $in4,$x40,$inp
2583 subi $len,$len,0x60
2584 vxor $twk4,$tweak,$rndkey0
2585 vsrab $tmp,$tweak,$seven # next tweak value
2586 vaddubm $tweak,$tweak,$tweak
2587 vsldoi $tmp,$tmp,$tmp,15
2588 le?vperm $in4,$in4,$in4,$leperm
2589 vand $tmp,$tmp,$eighty7
2590 vxor $out4,$in4,$twk4
2591 vxor $tweak,$tweak,$tmp
2592
2593 lvx_u $in5,$x50,$inp
2594 addi $inp,$inp,0x60
2595 vxor $twk5,$tweak,$rndkey0
2596 vsrab $tmp,$tweak,$seven # next tweak value
2597 vaddubm $tweak,$tweak,$tweak
2598 vsldoi $tmp,$tmp,$tmp,15
2599 le?vperm $in5,$in5,$in5,$leperm
2600 vand $tmp,$tmp,$eighty7
2601 vxor $out5,$in5,$twk5
2602 vxor $tweak,$tweak,$tmp
2603
2604 vxor v31,v31,$rndkey0
2605 mtctr $rounds
2606 b Loop_xts_enc6x
2607
2608 .align 5
2609 Loop_xts_enc6x:
2610 vcipher $out0,$out0,v24
2611 vcipher $out1,$out1,v24
2612 vcipher $out2,$out2,v24
2613 vcipher $out3,$out3,v24
2614 vcipher $out4,$out4,v24
2615 vcipher $out5,$out5,v24
2616 lvx v24,$x20,$key_ # round[3]
2617 addi $key_,$key_,0x20
2618
2619 vcipher $out0,$out0,v25
2620 vcipher $out1,$out1,v25
2621 vcipher $out2,$out2,v25
2622 vcipher $out3,$out3,v25
2623 vcipher $out4,$out4,v25
2624 vcipher $out5,$out5,v25
2625 lvx v25,$x10,$key_ # round[4]
2626 bdnz Loop_xts_enc6x
2627
2628 subic $len,$len,96 # $len-=96
2629 vxor $in0,$twk0,v31 # xor with last round key
2630 vcipher $out0,$out0,v24
2631 vcipher $out1,$out1,v24
2632 vsrab $tmp,$tweak,$seven # next tweak value
2633 vxor $twk0,$tweak,$rndkey0
2634 vaddubm $tweak,$tweak,$tweak
2635 vcipher $out2,$out2,v24
2636 vcipher $out3,$out3,v24
2637 vsldoi $tmp,$tmp,$tmp,15
2638 vcipher $out4,$out4,v24
2639 vcipher $out5,$out5,v24
2640
2641 subfe. r0,r0,r0 # borrow?-1:0
2642 vand $tmp,$tmp,$eighty7
2643 vcipher $out0,$out0,v25
2644 vcipher $out1,$out1,v25
2645 vxor $tweak,$tweak,$tmp
2646 vcipher $out2,$out2,v25
2647 vcipher $out3,$out3,v25
2648 vxor $in1,$twk1,v31
2649 vsrab $tmp,$tweak,$seven # next tweak value
2650 vxor $twk1,$tweak,$rndkey0
2651 vcipher $out4,$out4,v25
2652 vcipher $out5,$out5,v25
2653
2654 and r0,r0,$len
2655 vaddubm $tweak,$tweak,$tweak
2656 vsldoi $tmp,$tmp,$tmp,15
2657 vcipher $out0,$out0,v26
2658 vcipher $out1,$out1,v26
2659 vand $tmp,$tmp,$eighty7
2660 vcipher $out2,$out2,v26
2661 vcipher $out3,$out3,v26
2662 vxor $tweak,$tweak,$tmp
2663 vcipher $out4,$out4,v26
2664 vcipher $out5,$out5,v26
2665
2666 add $inp,$inp,r0 # $inp is adjusted in such
2667 # way that at exit from the
2668 # loop inX-in5 are loaded
2669 # with last "words"
2670 vxor $in2,$twk2,v31
2671 vsrab $tmp,$tweak,$seven # next tweak value
2672 vxor $twk2,$tweak,$rndkey0
2673 vaddubm $tweak,$tweak,$tweak
2674 vcipher $out0,$out0,v27
2675 vcipher $out1,$out1,v27
2676 vsldoi $tmp,$tmp,$tmp,15
2677 vcipher $out2,$out2,v27
2678 vcipher $out3,$out3,v27
2679 vand $tmp,$tmp,$eighty7
2680 vcipher $out4,$out4,v27
2681 vcipher $out5,$out5,v27
2682
2683 addi $key_,$sp,$FRAME+15 # rewind $key_
2684 vxor $tweak,$tweak,$tmp
2685 vcipher $out0,$out0,v28
2686 vcipher $out1,$out1,v28
2687 vxor $in3,$twk3,v31
2688 vsrab $tmp,$tweak,$seven # next tweak value
2689 vxor $twk3,$tweak,$rndkey0
2690 vcipher $out2,$out2,v28
2691 vcipher $out3,$out3,v28
2692 vaddubm $tweak,$tweak,$tweak
2693 vsldoi $tmp,$tmp,$tmp,15
2694 vcipher $out4,$out4,v28
2695 vcipher $out5,$out5,v28
2696 lvx v24,$x00,$key_ # re-pre-load round[1]
2697 vand $tmp,$tmp,$eighty7
2698
2699 vcipher $out0,$out0,v29
2700 vcipher $out1,$out1,v29
2701 vxor $tweak,$tweak,$tmp
2702 vcipher $out2,$out2,v29
2703 vcipher $out3,$out3,v29
2704 vxor $in4,$twk4,v31
2705 vsrab $tmp,$tweak,$seven # next tweak value
2706 vxor $twk4,$tweak,$rndkey0
2707 vcipher $out4,$out4,v29
2708 vcipher $out5,$out5,v29
2709 lvx v25,$x10,$key_ # re-pre-load round[2]
2710 vaddubm $tweak,$tweak,$tweak
2711 vsldoi $tmp,$tmp,$tmp,15
2712
2713 vcipher $out0,$out0,v30
2714 vcipher $out1,$out1,v30
2715 vand $tmp,$tmp,$eighty7
2716 vcipher $out2,$out2,v30
2717 vcipher $out3,$out3,v30
2718 vxor $tweak,$tweak,$tmp
2719 vcipher $out4,$out4,v30
2720 vcipher $out5,$out5,v30
2721 vxor $in5,$twk5,v31
2722 vsrab $tmp,$tweak,$seven # next tweak value
2723 vxor $twk5,$tweak,$rndkey0
2724
2725 vcipherlast $out0,$out0,$in0
2726 lvx_u $in0,$x00,$inp # load next input block
2727 vaddubm $tweak,$tweak,$tweak
2728 vsldoi $tmp,$tmp,$tmp,15
2729 vcipherlast $out1,$out1,$in1
2730 lvx_u $in1,$x10,$inp
2731 vcipherlast $out2,$out2,$in2
2732 le?vperm $in0,$in0,$in0,$leperm
2733 lvx_u $in2,$x20,$inp
2734 vand $tmp,$tmp,$eighty7
2735 vcipherlast $out3,$out3,$in3
2736 le?vperm $in1,$in1,$in1,$leperm
2737 lvx_u $in3,$x30,$inp
2738 vcipherlast $out4,$out4,$in4
2739 le?vperm $in2,$in2,$in2,$leperm
2740 lvx_u $in4,$x40,$inp
2741 vxor $tweak,$tweak,$tmp
2742 vcipherlast $tmp,$out5,$in5 # last block might be needed
2743 # in stealing mode
2744 le?vperm $in3,$in3,$in3,$leperm
2745 lvx_u $in5,$x50,$inp
2746 addi $inp,$inp,0x60
2747 le?vperm $in4,$in4,$in4,$leperm
2748 le?vperm $in5,$in5,$in5,$leperm
2749
2750 le?vperm $out0,$out0,$out0,$leperm
2751 le?vperm $out1,$out1,$out1,$leperm
2752 stvx_u $out0,$x00,$out # store output
2753 vxor $out0,$in0,$twk0
2754 le?vperm $out2,$out2,$out2,$leperm
2755 stvx_u $out1,$x10,$out
2756 vxor $out1,$in1,$twk1
2757 le?vperm $out3,$out3,$out3,$leperm
2758 stvx_u $out2,$x20,$out
2759 vxor $out2,$in2,$twk2
2760 le?vperm $out4,$out4,$out4,$leperm
2761 stvx_u $out3,$x30,$out
2762 vxor $out3,$in3,$twk3
2763 le?vperm $out5,$tmp,$tmp,$leperm
2764 stvx_u $out4,$x40,$out
2765 vxor $out4,$in4,$twk4
2766 le?stvx_u $out5,$x50,$out
2767 be?stvx_u $tmp, $x50,$out
2768 vxor $out5,$in5,$twk5
2769 addi $out,$out,0x60
2770
2771 mtctr $rounds
2772 beq Loop_xts_enc6x # did $len-=96 borrow?
2773
2774 addic. $len,$len,0x60
2775 beq Lxts_enc6x_zero
2776 cmpwi $len,0x20
2777 blt Lxts_enc6x_one
2778 nop
2779 beq Lxts_enc6x_two
2780 cmpwi $len,0x40
2781 blt Lxts_enc6x_three
2782 nop
2783 beq Lxts_enc6x_four
2784
2785 Lxts_enc6x_five:
2786 vxor $out0,$in1,$twk0
2787 vxor $out1,$in2,$twk1
2788 vxor $out2,$in3,$twk2
2789 vxor $out3,$in4,$twk3
2790 vxor $out4,$in5,$twk4
2791
2792 bl _aesp8_xts_enc5x
2793
2794 le?vperm $out0,$out0,$out0,$leperm
2795 vmr $twk0,$twk5 # unused tweak
2796 le?vperm $out1,$out1,$out1,$leperm
2797 stvx_u $out0,$x00,$out # store output
2798 le?vperm $out2,$out2,$out2,$leperm
2799 stvx_u $out1,$x10,$out
2800 le?vperm $out3,$out3,$out3,$leperm
2801 stvx_u $out2,$x20,$out
2802 vxor $tmp,$out4,$twk5 # last block prep for stealing
2803 le?vperm $out4,$out4,$out4,$leperm
2804 stvx_u $out3,$x30,$out
2805 stvx_u $out4,$x40,$out
2806 addi $out,$out,0x50
2807 bne Lxts_enc6x_steal
2808 b Lxts_enc6x_done
2809
2810 .align 4
2811 Lxts_enc6x_four:
2812 vxor $out0,$in2,$twk0
2813 vxor $out1,$in3,$twk1
2814 vxor $out2,$in4,$twk2
2815 vxor $out3,$in5,$twk3
2816 vxor $out4,$out4,$out4
2817
2818 bl _aesp8_xts_enc5x
2819
2820 le?vperm $out0,$out0,$out0,$leperm
2821 vmr $twk0,$twk4 # unused tweak
2822 le?vperm $out1,$out1,$out1,$leperm
2823 stvx_u $out0,$x00,$out # store output
2824 le?vperm $out2,$out2,$out2,$leperm
2825 stvx_u $out1,$x10,$out
2826 vxor $tmp,$out3,$twk4 # last block prep for stealing
2827 le?vperm $out3,$out3,$out3,$leperm
2828 stvx_u $out2,$x20,$out
2829 stvx_u $out3,$x30,$out
2830 addi $out,$out,0x40
2831 bne Lxts_enc6x_steal
2832 b Lxts_enc6x_done
2833
2834 .align 4
2835 Lxts_enc6x_three:
2836 vxor $out0,$in3,$twk0
2837 vxor $out1,$in4,$twk1
2838 vxor $out2,$in5,$twk2
2839 vxor $out3,$out3,$out3
2840 vxor $out4,$out4,$out4
2841
2842 bl _aesp8_xts_enc5x
2843
2844 le?vperm $out0,$out0,$out0,$leperm
2845 vmr $twk0,$twk3 # unused tweak
2846 le?vperm $out1,$out1,$out1,$leperm
2847 stvx_u $out0,$x00,$out # store output
2848 vxor $tmp,$out2,$twk3 # last block prep for stealing
2849 le?vperm $out2,$out2,$out2,$leperm
2850 stvx_u $out1,$x10,$out
2851 stvx_u $out2,$x20,$out
2852 addi $out,$out,0x30
2853 bne Lxts_enc6x_steal
2854 b Lxts_enc6x_done
2855
2856 .align 4
2857 Lxts_enc6x_two:
2858 vxor $out0,$in4,$twk0
2859 vxor $out1,$in5,$twk1
2860 vxor $out2,$out2,$out2
2861 vxor $out3,$out3,$out3
2862 vxor $out4,$out4,$out4
2863
2864 bl _aesp8_xts_enc5x
2865
2866 le?vperm $out0,$out0,$out0,$leperm
2867 vmr $twk0,$twk2 # unused tweak
2868 vxor $tmp,$out1,$twk2 # last block prep for stealing
2869 le?vperm $out1,$out1,$out1,$leperm
2870 stvx_u $out0,$x00,$out # store output
2871 stvx_u $out1,$x10,$out
2872 addi $out,$out,0x20
2873 bne Lxts_enc6x_steal
2874 b Lxts_enc6x_done
2875
2876 .align 4
2877 Lxts_enc6x_one:
2878 vxor $out0,$in5,$twk0
2879 nop
2880 Loop_xts_enc1x:
2881 vcipher $out0,$out0,v24
2882 lvx v24,$x20,$key_ # round[3]
2883 addi $key_,$key_,0x20
2884
2885 vcipher $out0,$out0,v25
2886 lvx v25,$x10,$key_ # round[4]
2887 bdnz Loop_xts_enc1x
2888
2889 add $inp,$inp,$taillen
2890 cmpwi $taillen,0
2891 vcipher $out0,$out0,v24
2892
2893 subi $inp,$inp,16
2894 vcipher $out0,$out0,v25
2895
2896 lvsr $inpperm,0,$taillen
2897 vcipher $out0,$out0,v26
2898
2899 lvx_u $in0,0,$inp
2900 vcipher $out0,$out0,v27
2901
2902 addi $key_,$sp,$FRAME+15 # rewind $key_
2903 vcipher $out0,$out0,v28
2904 lvx v24,$x00,$key_ # re-pre-load round[1]
2905
2906 vcipher $out0,$out0,v29
2907 lvx v25,$x10,$key_ # re-pre-load round[2]
2908 vxor $twk0,$twk0,v31
2909
2910 le?vperm $in0,$in0,$in0,$leperm
2911 vcipher $out0,$out0,v30
2912
2913 vperm $in0,$in0,$in0,$inpperm
2914 vcipherlast $out0,$out0,$twk0
2915
2916 vmr $twk0,$twk1 # unused tweak
2917 vxor $tmp,$out0,$twk1 # last block prep for stealing
2918 le?vperm $out0,$out0,$out0,$leperm
2919 stvx_u $out0,$x00,$out # store output
2920 addi $out,$out,0x10
2921 bne Lxts_enc6x_steal
2922 b Lxts_enc6x_done
2923
2924 .align 4
2925 Lxts_enc6x_zero:
2926 cmpwi $taillen,0
2927 beq Lxts_enc6x_done
2928
2929 add $inp,$inp,$taillen
2930 subi $inp,$inp,16
2931 lvx_u $in0,0,$inp
2932 lvsr $inpperm,0,$taillen # $in5 is no more
2933 le?vperm $in0,$in0,$in0,$leperm
2934 vperm $in0,$in0,$in0,$inpperm
2935 vxor $tmp,$tmp,$twk0
2936 Lxts_enc6x_steal:
2937 vxor $in0,$in0,$twk0
2938 vxor $out0,$out0,$out0
2939 vspltisb $out1,-1
2940 vperm $out0,$out0,$out1,$inpperm
2941 vsel $out0,$in0,$tmp,$out0 # $tmp is last block, remember?
2942
2943 subi r30,$out,17
2944 subi $out,$out,16
2945 mtctr $taillen
2946 Loop_xts_enc6x_steal:
2947 lbzu r0,1(r30)
2948 stb r0,16(r30)
2949 bdnz Loop_xts_enc6x_steal
2950
2951 li $taillen,0
2952 mtctr $rounds
2953 b Loop_xts_enc1x # one more time...
2954
2955 .align 4
2956 Lxts_enc6x_done:
2957 ${UCMP}i $ivp,0
2958 beq Lxts_enc6x_ret
2959
2960 vxor $tweak,$twk0,$rndkey0
2961 le?vperm $tweak,$tweak,$tweak,$leperm
2962 stvx_u $tweak,0,$ivp
2963
2964 Lxts_enc6x_ret:
2965 mtlr r11
2966 li r10,`$FRAME+15`
2967 li r11,`$FRAME+31`
2968 stvx $seven,r10,$sp # wipe copies of round keys
2969 addi r10,r10,32
2970 stvx $seven,r11,$sp
2971 addi r11,r11,32
2972 stvx $seven,r10,$sp
2973 addi r10,r10,32
2974 stvx $seven,r11,$sp
2975 addi r11,r11,32
2976 stvx $seven,r10,$sp
2977 addi r10,r10,32
2978 stvx $seven,r11,$sp
2979 addi r11,r11,32
2980 stvx $seven,r10,$sp
2981 addi r10,r10,32
2982 stvx $seven,r11,$sp
2983 addi r11,r11,32
2984
2985 mtspr 256,$vrsave
2986 lvx v20,r10,$sp # ABI says so
2987 addi r10,r10,32
2988 lvx v21,r11,$sp
2989 addi r11,r11,32
2990 lvx v22,r10,$sp
2991 addi r10,r10,32
2992 lvx v23,r11,$sp
2993 addi r11,r11,32
2994 lvx v24,r10,$sp
2995 addi r10,r10,32
2996 lvx v25,r11,$sp
2997 addi r11,r11,32
2998 lvx v26,r10,$sp
2999 addi r10,r10,32
3000 lvx v27,r11,$sp
3001 addi r11,r11,32
3002 lvx v28,r10,$sp
3003 addi r10,r10,32
3004 lvx v29,r11,$sp
3005 addi r11,r11,32
3006 lvx v30,r10,$sp
3007 lvx v31,r11,$sp
3008 $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
3009 $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
3010 $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
3011 $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
3012 $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
3013 $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
3014 addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
3015 blr
3016 .long 0
3017 .byte 0,12,0x04,1,0x80,6,6,0
3018 .long 0
3019
3020 .align 5
3021 _aesp8_xts_enc5x:
3022 vcipher $out0,$out0,v24
3023 vcipher $out1,$out1,v24
3024 vcipher $out2,$out2,v24
3025 vcipher $out3,$out3,v24
3026 vcipher $out4,$out4,v24
3027 lvx v24,$x20,$key_ # round[3]
3028 addi $key_,$key_,0x20
3029
3030 vcipher $out0,$out0,v25
3031 vcipher $out1,$out1,v25
3032 vcipher $out2,$out2,v25
3033 vcipher $out3,$out3,v25
3034 vcipher $out4,$out4,v25
3035 lvx v25,$x10,$key_ # round[4]
3036 bdnz _aesp8_xts_enc5x
3037
3038 add $inp,$inp,$taillen
3039 cmpwi $taillen,0
3040 vcipher $out0,$out0,v24
3041 vcipher $out1,$out1,v24
3042 vcipher $out2,$out2,v24
3043 vcipher $out3,$out3,v24
3044 vcipher $out4,$out4,v24
3045
3046 subi $inp,$inp,16
3047 vcipher $out0,$out0,v25
3048 vcipher $out1,$out1,v25
3049 vcipher $out2,$out2,v25
3050 vcipher $out3,$out3,v25
3051 vcipher $out4,$out4,v25
3052 vxor $twk0,$twk0,v31
3053
3054 vcipher $out0,$out0,v26
3055 lvsr $inpperm,r0,$taillen # $in5 is no more
3056 vcipher $out1,$out1,v26
3057 vcipher $out2,$out2,v26
3058 vcipher $out3,$out3,v26
3059 vcipher $out4,$out4,v26
3060 vxor $in1,$twk1,v31
3061
3062 vcipher $out0,$out0,v27
3063 lvx_u $in0,0,$inp
3064 vcipher $out1,$out1,v27
3065 vcipher $out2,$out2,v27
3066 vcipher $out3,$out3,v27
3067 vcipher $out4,$out4,v27
3068 vxor $in2,$twk2,v31
3069
3070 addi $key_,$sp,$FRAME+15 # rewind $key_
3071 vcipher $out0,$out0,v28
3072 vcipher $out1,$out1,v28
3073 vcipher $out2,$out2,v28
3074 vcipher $out3,$out3,v28
3075 vcipher $out4,$out4,v28
3076 lvx v24,$x00,$key_ # re-pre-load round[1]
3077 vxor $in3,$twk3,v31
3078
3079 vcipher $out0,$out0,v29
3080 le?vperm $in0,$in0,$in0,$leperm
3081 vcipher $out1,$out1,v29
3082 vcipher $out2,$out2,v29
3083 vcipher $out3,$out3,v29
3084 vcipher $out4,$out4,v29
3085 lvx v25,$x10,$key_ # re-pre-load round[2]
3086 vxor $in4,$twk4,v31
3087
3088 vcipher $out0,$out0,v30
3089 vperm $in0,$in0,$in0,$inpperm
3090 vcipher $out1,$out1,v30
3091 vcipher $out2,$out2,v30
3092 vcipher $out3,$out3,v30
3093 vcipher $out4,$out4,v30
3094
3095 vcipherlast $out0,$out0,$twk0
3096 vcipherlast $out1,$out1,$in1
3097 vcipherlast $out2,$out2,$in2
3098 vcipherlast $out3,$out3,$in3
3099 vcipherlast $out4,$out4,$in4
3100 blr
3101 .long 0
3102 .byte 0,12,0x14,0,0,0,0,0
3103
3104 .align 5
3105 _aesp8_xts_decrypt6x:
3106 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
3107 mflr r11
3108 li r7,`$FRAME+8*16+15`
3109 li r3,`$FRAME+8*16+31`
3110 $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
3111 stvx v20,r7,$sp # ABI says so
3112 addi r7,r7,32
3113 stvx v21,r3,$sp
3114 addi r3,r3,32
3115 stvx v22,r7,$sp
3116 addi r7,r7,32
3117 stvx v23,r3,$sp
3118 addi r3,r3,32
3119 stvx v24,r7,$sp
3120 addi r7,r7,32
3121 stvx v25,r3,$sp
3122 addi r3,r3,32
3123 stvx v26,r7,$sp
3124 addi r7,r7,32
3125 stvx v27,r3,$sp
3126 addi r3,r3,32
3127 stvx v28,r7,$sp
3128 addi r7,r7,32
3129 stvx v29,r3,$sp
3130 addi r3,r3,32
3131 stvx v30,r7,$sp
3132 stvx v31,r3,$sp
3133 li r0,-1
3134 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
3135 li $x10,0x10
3136 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
3137 li $x20,0x20
3138 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
3139 li $x30,0x30
3140 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
3141 li $x40,0x40
3142 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
3143 li $x50,0x50
3144 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
3145 li $x60,0x60
3146 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
3147 li $x70,0x70
3148 mtspr 256,r0
3149
3150 subi $rounds,$rounds,3 # -4 in total
3151
3152 lvx $rndkey0,$x00,$key1 # load key schedule
3153 lvx v30,$x10,$key1
3154 addi $key1,$key1,0x20
3155 lvx v31,$x00,$key1
3156 ?vperm $rndkey0,$rndkey0,v30,$keyperm
3157 addi $key_,$sp,$FRAME+15
3158 mtctr $rounds
3159
3160 Load_xts_dec_key:
3161 ?vperm v24,v30,v31,$keyperm
3162 lvx v30,$x10,$key1
3163 addi $key1,$key1,0x20
3164 stvx v24,$x00,$key_ # off-load round[1]
3165 ?vperm v25,v31,v30,$keyperm
3166 lvx v31,$x00,$key1
3167 stvx v25,$x10,$key_ # off-load round[2]
3168 addi $key_,$key_,0x20
3169 bdnz Load_xts_dec_key
3170
3171 lvx v26,$x10,$key1
3172 ?vperm v24,v30,v31,$keyperm
3173 lvx v27,$x20,$key1
3174 stvx v24,$x00,$key_ # off-load round[3]
3175 ?vperm v25,v31,v26,$keyperm
3176 lvx v28,$x30,$key1
3177 stvx v25,$x10,$key_ # off-load round[4]
3178 addi $key_,$sp,$FRAME+15 # rewind $key_
3179 ?vperm v26,v26,v27,$keyperm
3180 lvx v29,$x40,$key1
3181 ?vperm v27,v27,v28,$keyperm
3182 lvx v30,$x50,$key1
3183 ?vperm v28,v28,v29,$keyperm
3184 lvx v31,$x60,$key1
3185 ?vperm v29,v29,v30,$keyperm
3186 lvx $twk5,$x70,$key1 # borrow $twk5
3187 ?vperm v30,v30,v31,$keyperm
3188 lvx v24,$x00,$key_ # pre-load round[1]
3189 ?vperm v31,v31,$twk5,$keyperm
3190 lvx v25,$x10,$key_ # pre-load round[2]
3191
3192 vperm $in0,$inout,$inptail,$inpperm
3193 subi $inp,$inp,31 # undo "caller"
3194 vxor $twk0,$tweak,$rndkey0
3195 vsrab $tmp,$tweak,$seven # next tweak value
3196 vaddubm $tweak,$tweak,$tweak
3197 vsldoi $tmp,$tmp,$tmp,15
3198 vand $tmp,$tmp,$eighty7
3199 vxor $out0,$in0,$twk0
3200 vxor $tweak,$tweak,$tmp
3201
3202 lvx_u $in1,$x10,$inp
3203 vxor $twk1,$tweak,$rndkey0
3204 vsrab $tmp,$tweak,$seven # next tweak value
3205 vaddubm $tweak,$tweak,$tweak
3206 vsldoi $tmp,$tmp,$tmp,15
3207 le?vperm $in1,$in1,$in1,$leperm
3208 vand $tmp,$tmp,$eighty7
3209 vxor $out1,$in1,$twk1
3210 vxor $tweak,$tweak,$tmp
3211
3212 lvx_u $in2,$x20,$inp
3213 andi. $taillen,$len,15
3214 vxor $twk2,$tweak,$rndkey0
3215 vsrab $tmp,$tweak,$seven # next tweak value
3216 vaddubm $tweak,$tweak,$tweak
3217 vsldoi $tmp,$tmp,$tmp,15
3218 le?vperm $in2,$in2,$in2,$leperm
3219 vand $tmp,$tmp,$eighty7
3220 vxor $out2,$in2,$twk2
3221 vxor $tweak,$tweak,$tmp
3222
3223 lvx_u $in3,$x30,$inp
3224 sub $len,$len,$taillen
3225 vxor $twk3,$tweak,$rndkey0
3226 vsrab $tmp,$tweak,$seven # next tweak value
3227 vaddubm $tweak,$tweak,$tweak
3228 vsldoi $tmp,$tmp,$tmp,15
3229 le?vperm $in3,$in3,$in3,$leperm
3230 vand $tmp,$tmp,$eighty7
3231 vxor $out3,$in3,$twk3
3232 vxor $tweak,$tweak,$tmp
3233
3234 lvx_u $in4,$x40,$inp
3235 subi $len,$len,0x60
3236 vxor $twk4,$tweak,$rndkey0
3237 vsrab $tmp,$tweak,$seven # next tweak value
3238 vaddubm $tweak,$tweak,$tweak
3239 vsldoi $tmp,$tmp,$tmp,15
3240 le?vperm $in4,$in4,$in4,$leperm
3241 vand $tmp,$tmp,$eighty7
3242 vxor $out4,$in4,$twk4
3243 vxor $tweak,$tweak,$tmp
3244
3245 lvx_u $in5,$x50,$inp
3246 addi $inp,$inp,0x60
3247 vxor $twk5,$tweak,$rndkey0
3248 vsrab $tmp,$tweak,$seven # next tweak value
3249 vaddubm $tweak,$tweak,$tweak
3250 vsldoi $tmp,$tmp,$tmp,15
3251 le?vperm $in5,$in5,$in5,$leperm
3252 vand $tmp,$tmp,$eighty7
3253 vxor $out5,$in5,$twk5
3254 vxor $tweak,$tweak,$tmp
3255
3256 vxor v31,v31,$rndkey0
3257 mtctr $rounds
3258 b Loop_xts_dec6x
3259
3260 .align 5
3261 Loop_xts_dec6x:
3262 vncipher $out0,$out0,v24
3263 vncipher $out1,$out1,v24
3264 vncipher $out2,$out2,v24
3265 vncipher $out3,$out3,v24
3266 vncipher $out4,$out4,v24
3267 vncipher $out5,$out5,v24
3268 lvx v24,$x20,$key_ # round[3]
3269 addi $key_,$key_,0x20
3270
3271 vncipher $out0,$out0,v25
3272 vncipher $out1,$out1,v25
3273 vncipher $out2,$out2,v25
3274 vncipher $out3,$out3,v25
3275 vncipher $out4,$out4,v25
3276 vncipher $out5,$out5,v25
3277 lvx v25,$x10,$key_ # round[4]
3278 bdnz Loop_xts_dec6x
3279
3280 subic $len,$len,96 # $len-=96
3281 vxor $in0,$twk0,v31 # xor with last round key
3282 vncipher $out0,$out0,v24
3283 vncipher $out1,$out1,v24
3284 vsrab $tmp,$tweak,$seven # next tweak value
3285 vxor $twk0,$tweak,$rndkey0
3286 vaddubm $tweak,$tweak,$tweak
3287 vncipher $out2,$out2,v24
3288 vncipher $out3,$out3,v24
3289 vsldoi $tmp,$tmp,$tmp,15
3290 vncipher $out4,$out4,v24
3291 vncipher $out5,$out5,v24
3292
3293 subfe. r0,r0,r0 # borrow?-1:0
3294 vand $tmp,$tmp,$eighty7
3295 vncipher $out0,$out0,v25
3296 vncipher $out1,$out1,v25
3297 vxor $tweak,$tweak,$tmp
3298 vncipher $out2,$out2,v25
3299 vncipher $out3,$out3,v25
3300 vxor $in1,$twk1,v31
3301 vsrab $tmp,$tweak,$seven # next tweak value
3302 vxor $twk1,$tweak,$rndkey0
3303 vncipher $out4,$out4,v25
3304 vncipher $out5,$out5,v25
3305
3306 and r0,r0,$len
3307 vaddubm $tweak,$tweak,$tweak
3308 vsldoi $tmp,$tmp,$tmp,15
3309 vncipher $out0,$out0,v26
3310 vncipher $out1,$out1,v26
3311 vand $tmp,$tmp,$eighty7
3312 vncipher $out2,$out2,v26
3313 vncipher $out3,$out3,v26
3314 vxor $tweak,$tweak,$tmp
3315 vncipher $out4,$out4,v26
3316 vncipher $out5,$out5,v26
3317
3318 add $inp,$inp,r0 # $inp is adjusted in such
3319 # way that at exit from the
3320 # loop inX-in5 are loaded
3321 # with last "words"
3322 vxor $in2,$twk2,v31
3323 vsrab $tmp,$tweak,$seven # next tweak value
3324 vxor $twk2,$tweak,$rndkey0
3325 vaddubm $tweak,$tweak,$tweak
3326 vncipher $out0,$out0,v27
3327 vncipher $out1,$out1,v27
3328 vsldoi $tmp,$tmp,$tmp,15
3329 vncipher $out2,$out2,v27
3330 vncipher $out3,$out3,v27
3331 vand $tmp,$tmp,$eighty7
3332 vncipher $out4,$out4,v27
3333 vncipher $out5,$out5,v27
3334
3335 addi $key_,$sp,$FRAME+15 # rewind $key_
3336 vxor $tweak,$tweak,$tmp
3337 vncipher $out0,$out0,v28
3338 vncipher $out1,$out1,v28
3339 vxor $in3,$twk3,v31
3340 vsrab $tmp,$tweak,$seven # next tweak value
3341 vxor $twk3,$tweak,$rndkey0
3342 vncipher $out2,$out2,v28
3343 vncipher $out3,$out3,v28
3344 vaddubm $tweak,$tweak,$tweak
3345 vsldoi $tmp,$tmp,$tmp,15
3346 vncipher $out4,$out4,v28
3347 vncipher $out5,$out5,v28
3348 lvx v24,$x00,$key_ # re-pre-load round[1]
3349 vand $tmp,$tmp,$eighty7
3350
3351 vncipher $out0,$out0,v29
3352 vncipher $out1,$out1,v29
3353 vxor $tweak,$tweak,$tmp
3354 vncipher $out2,$out2,v29
3355 vncipher $out3,$out3,v29
3356 vxor $in4,$twk4,v31
3357 vsrab $tmp,$tweak,$seven # next tweak value
3358 vxor $twk4,$tweak,$rndkey0
3359 vncipher $out4,$out4,v29
3360 vncipher $out5,$out5,v29
3361 lvx v25,$x10,$key_ # re-pre-load round[2]
3362 vaddubm $tweak,$tweak,$tweak
3363 vsldoi $tmp,$tmp,$tmp,15
3364
3365 vncipher $out0,$out0,v30
3366 vncipher $out1,$out1,v30
3367 vand $tmp,$tmp,$eighty7
3368 vncipher $out2,$out2,v30
3369 vncipher $out3,$out3,v30
3370 vxor $tweak,$tweak,$tmp
3371 vncipher $out4,$out4,v30
3372 vncipher $out5,$out5,v30
3373 vxor $in5,$twk5,v31
3374 vsrab $tmp,$tweak,$seven # next tweak value
3375 vxor $twk5,$tweak,$rndkey0
3376
3377 vncipherlast $out0,$out0,$in0
3378 lvx_u $in0,$x00,$inp # load next input block
3379 vaddubm $tweak,$tweak,$tweak
3380 vsldoi $tmp,$tmp,$tmp,15
3381 vncipherlast $out1,$out1,$in1
3382 lvx_u $in1,$x10,$inp
3383 vncipherlast $out2,$out2,$in2
3384 le?vperm $in0,$in0,$in0,$leperm
3385 lvx_u $in2,$x20,$inp
3386 vand $tmp,$tmp,$eighty7
3387 vncipherlast $out3,$out3,$in3
3388 le?vperm $in1,$in1,$in1,$leperm
3389 lvx_u $in3,$x30,$inp
3390 vncipherlast $out4,$out4,$in4
3391 le?vperm $in2,$in2,$in2,$leperm
3392 lvx_u $in4,$x40,$inp
3393 vxor $tweak,$tweak,$tmp
3394 vncipherlast $out5,$out5,$in5
3395 le?vperm $in3,$in3,$in3,$leperm
3396 lvx_u $in5,$x50,$inp
3397 addi $inp,$inp,0x60
3398 le?vperm $in4,$in4,$in4,$leperm
3399 le?vperm $in5,$in5,$in5,$leperm
3400
3401 le?vperm $out0,$out0,$out0,$leperm
3402 le?vperm $out1,$out1,$out1,$leperm
3403 stvx_u $out0,$x00,$out # store output
3404 vxor $out0,$in0,$twk0
3405 le?vperm $out2,$out2,$out2,$leperm
3406 stvx_u $out1,$x10,$out
3407 vxor $out1,$in1,$twk1
3408 le?vperm $out3,$out3,$out3,$leperm
3409 stvx_u $out2,$x20,$out
3410 vxor $out2,$in2,$twk2
3411 le?vperm $out4,$out4,$out4,$leperm
3412 stvx_u $out3,$x30,$out
3413 vxor $out3,$in3,$twk3
3414 le?vperm $out5,$out5,$out5,$leperm
3415 stvx_u $out4,$x40,$out
3416 vxor $out4,$in4,$twk4
3417 stvx_u $out5,$x50,$out
3418 vxor $out5,$in5,$twk5
3419 addi $out,$out,0x60
3420
3421 mtctr $rounds
3422 beq Loop_xts_dec6x # did $len-=96 borrow?
3423
3424 addic. $len,$len,0x60
3425 beq Lxts_dec6x_zero
3426 cmpwi $len,0x20
3427 blt Lxts_dec6x_one
3428 nop
3429 beq Lxts_dec6x_two
3430 cmpwi $len,0x40
3431 blt Lxts_dec6x_three
3432 nop
3433 beq Lxts_dec6x_four
3434
3435 Lxts_dec6x_five:
3436 vxor $out0,$in1,$twk0
3437 vxor $out1,$in2,$twk1
3438 vxor $out2,$in3,$twk2
3439 vxor $out3,$in4,$twk3
3440 vxor $out4,$in5,$twk4
3441
3442 bl _aesp8_xts_dec5x
3443
3444 le?vperm $out0,$out0,$out0,$leperm
3445 vmr $twk0,$twk5 # unused tweak
3446 vxor $twk1,$tweak,$rndkey0
3447 le?vperm $out1,$out1,$out1,$leperm
3448 stvx_u $out0,$x00,$out # store output
3449 vxor $out0,$in0,$twk1
3450 le?vperm $out2,$out2,$out2,$leperm
3451 stvx_u $out1,$x10,$out
3452 le?vperm $out3,$out3,$out3,$leperm
3453 stvx_u $out2,$x20,$out
3454 le?vperm $out4,$out4,$out4,$leperm
3455 stvx_u $out3,$x30,$out
3456 stvx_u $out4,$x40,$out
3457 addi $out,$out,0x50
3458 bne Lxts_dec6x_steal
3459 b Lxts_dec6x_done
3460
3461 .align 4
3462 Lxts_dec6x_four:
3463 vxor $out0,$in2,$twk0
3464 vxor $out1,$in3,$twk1
3465 vxor $out2,$in4,$twk2
3466 vxor $out3,$in5,$twk3
3467 vxor $out4,$out4,$out4
3468
3469 bl _aesp8_xts_dec5x
3470
3471 le?vperm $out0,$out0,$out0,$leperm
3472 vmr $twk0,$twk4 # unused tweak
3473 vmr $twk1,$twk5
3474 le?vperm $out1,$out1,$out1,$leperm
3475 stvx_u $out0,$x00,$out # store output
3476 vxor $out0,$in0,$twk5
3477 le?vperm $out2,$out2,$out2,$leperm
3478 stvx_u $out1,$x10,$out
3479 le?vperm $out3,$out3,$out3,$leperm
3480 stvx_u $out2,$x20,$out
3481 stvx_u $out3,$x30,$out
3482 addi $out,$out,0x40
3483 bne Lxts_dec6x_steal
3484 b Lxts_dec6x_done
3485
3486 .align 4
3487 Lxts_dec6x_three:
3488 vxor $out0,$in3,$twk0
3489 vxor $out1,$in4,$twk1
3490 vxor $out2,$in5,$twk2
3491 vxor $out3,$out3,$out3
3492 vxor $out4,$out4,$out4
3493
3494 bl _aesp8_xts_dec5x
3495
3496 le?vperm $out0,$out0,$out0,$leperm
3497 vmr $twk0,$twk3 # unused tweak
3498 vmr $twk1,$twk4
3499 le?vperm $out1,$out1,$out1,$leperm
3500 stvx_u $out0,$x00,$out # store output
3501 vxor $out0,$in0,$twk4
3502 le?vperm $out2,$out2,$out2,$leperm
3503 stvx_u $out1,$x10,$out
3504 stvx_u $out2,$x20,$out
3505 addi $out,$out,0x30
3506 bne Lxts_dec6x_steal
3507 b Lxts_dec6x_done
3508
3509 .align 4
3510 Lxts_dec6x_two:
3511 vxor $out0,$in4,$twk0
3512 vxor $out1,$in5,$twk1
3513 vxor $out2,$out2,$out2
3514 vxor $out3,$out3,$out3
3515 vxor $out4,$out4,$out4
3516
3517 bl _aesp8_xts_dec5x
3518
3519 le?vperm $out0,$out0,$out0,$leperm
3520 vmr $twk0,$twk2 # unused tweak
3521 vmr $twk1,$twk3
3522 le?vperm $out1,$out1,$out1,$leperm
3523 stvx_u $out0,$x00,$out # store output
3524 vxor $out0,$in0,$twk3
3525 stvx_u $out1,$x10,$out
3526 addi $out,$out,0x20
3527 bne Lxts_dec6x_steal
3528 b Lxts_dec6x_done
3529
3530 .align 4
3531 Lxts_dec6x_one:
3532 vxor $out0,$in5,$twk0
3533 nop
3534 Loop_xts_dec1x:
3535 vncipher $out0,$out0,v24
3536 lvx v24,$x20,$key_ # round[3]
3537 addi $key_,$key_,0x20
3538
3539 vncipher $out0,$out0,v25
3540 lvx v25,$x10,$key_ # round[4]
3541 bdnz Loop_xts_dec1x
3542
3543 subi r0,$taillen,1
3544 vncipher $out0,$out0,v24
3545
3546 andi. r0,r0,16
3547 cmpwi $taillen,0
3548 vncipher $out0,$out0,v25
3549
3550 sub $inp,$inp,r0
3551 vncipher $out0,$out0,v26
3552
3553 lvx_u $in0,0,$inp
3554 vncipher $out0,$out0,v27
3555
3556 addi $key_,$sp,$FRAME+15 # rewind $key_
3557 vncipher $out0,$out0,v28
3558 lvx v24,$x00,$key_ # re-pre-load round[1]
3559
3560 vncipher $out0,$out0,v29
3561 lvx v25,$x10,$key_ # re-pre-load round[2]
3562 vxor $twk0,$twk0,v31
3563
3564 le?vperm $in0,$in0,$in0,$leperm
3565 vncipher $out0,$out0,v30
3566
3567 mtctr $rounds
3568 vncipherlast $out0,$out0,$twk0
3569
3570 vmr $twk0,$twk1 # unused tweak
3571 vmr $twk1,$twk2
3572 le?vperm $out0,$out0,$out0,$leperm
3573 stvx_u $out0,$x00,$out # store output
3574 addi $out,$out,0x10
3575 vxor $out0,$in0,$twk2
3576 bne Lxts_dec6x_steal
3577 b Lxts_dec6x_done
3578
3579 .align 4
3580 Lxts_dec6x_zero:
3581 cmpwi $taillen,0
3582 beq Lxts_dec6x_done
3583
3584 lvx_u $in0,0,$inp
3585 le?vperm $in0,$in0,$in0,$leperm
3586 vxor $out0,$in0,$twk1
3587 Lxts_dec6x_steal:
3588 vncipher $out0,$out0,v24
3589 lvx v24,$x20,$key_ # round[3]
3590 addi $key_,$key_,0x20
3591
3592 vncipher $out0,$out0,v25
3593 lvx v25,$x10,$key_ # round[4]
3594 bdnz Lxts_dec6x_steal
3595
3596 add $inp,$inp,$taillen
3597 vncipher $out0,$out0,v24
3598
3599 cmpwi $taillen,0
3600 vncipher $out0,$out0,v25
3601
3602 lvx_u $in0,0,$inp
3603 vncipher $out0,$out0,v26
3604
3605 lvsr $inpperm,0,$taillen # $in5 is no more
3606 vncipher $out0,$out0,v27
3607
3608 addi $key_,$sp,$FRAME+15 # rewind $key_
3609 vncipher $out0,$out0,v28
3610 lvx v24,$x00,$key_ # re-pre-load round[1]
3611
3612 vncipher $out0,$out0,v29
3613 lvx v25,$x10,$key_ # re-pre-load round[2]
3614 vxor $twk1,$twk1,v31
3615
3616 le?vperm $in0,$in0,$in0,$leperm
3617 vncipher $out0,$out0,v30
3618
3619 vperm $in0,$in0,$in0,$inpperm
3620 vncipherlast $tmp,$out0,$twk1
3621
3622 le?vperm $out0,$tmp,$tmp,$leperm
3623 le?stvx_u $out0,0,$out
3624 be?stvx_u $tmp,0,$out
3625
3626 vxor $out0,$out0,$out0
3627 vspltisb $out1,-1
3628 vperm $out0,$out0,$out1,$inpperm
3629 vsel $out0,$in0,$tmp,$out0
3630 vxor $out0,$out0,$twk0
3631
3632 subi r30,$out,1
3633 mtctr $taillen
3634 Loop_xts_dec6x_steal:
3635 lbzu r0,1(r30)
3636 stb r0,16(r30)
3637 bdnz Loop_xts_dec6x_steal
3638
3639 li $taillen,0
3640 mtctr $rounds
3641 b Loop_xts_dec1x # one more time...
3642
3643 .align 4
3644 Lxts_dec6x_done:
3645 ${UCMP}i $ivp,0
3646 beq Lxts_dec6x_ret
3647
3648 vxor $tweak,$twk0,$rndkey0
3649 le?vperm $tweak,$tweak,$tweak,$leperm
3650 stvx_u $tweak,0,$ivp
3651
3652 Lxts_dec6x_ret:
3653 mtlr r11
3654 li r10,`$FRAME+15`
3655 li r11,`$FRAME+31`
3656 stvx $seven,r10,$sp # wipe copies of round keys
3657 addi r10,r10,32
3658 stvx $seven,r11,$sp
3659 addi r11,r11,32
3660 stvx $seven,r10,$sp
3661 addi r10,r10,32
3662 stvx $seven,r11,$sp
3663 addi r11,r11,32
3664 stvx $seven,r10,$sp
3665 addi r10,r10,32
3666 stvx $seven,r11,$sp
3667 addi r11,r11,32
3668 stvx $seven,r10,$sp
3669 addi r10,r10,32
3670 stvx $seven,r11,$sp
3671 addi r11,r11,32
3672
3673 mtspr 256,$vrsave
3674 lvx v20,r10,$sp # ABI says so
3675 addi r10,r10,32
3676 lvx v21,r11,$sp
3677 addi r11,r11,32
3678 lvx v22,r10,$sp
3679 addi r10,r10,32
3680 lvx v23,r11,$sp
3681 addi r11,r11,32
3682 lvx v24,r10,$sp
3683 addi r10,r10,32
3684 lvx v25,r11,$sp
3685 addi r11,r11,32
3686 lvx v26,r10,$sp
3687 addi r10,r10,32
3688 lvx v27,r11,$sp
3689 addi r11,r11,32
3690 lvx v28,r10,$sp
3691 addi r10,r10,32
3692 lvx v29,r11,$sp
3693 addi r11,r11,32
3694 lvx v30,r10,$sp
3695 lvx v31,r11,$sp
3696 $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
3697 $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
3698 $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
3699 $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
3700 $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
3701 $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
3702 addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
3703 blr
3704 .long 0
3705 .byte 0,12,0x04,1,0x80,6,6,0
3706 .long 0
3707
3708 .align 5
3709 _aesp8_xts_dec5x:
3710 vncipher $out0,$out0,v24
3711 vncipher $out1,$out1,v24
3712 vncipher $out2,$out2,v24
3713 vncipher $out3,$out3,v24
3714 vncipher $out4,$out4,v24
3715 lvx v24,$x20,$key_ # round[3]
3716 addi $key_,$key_,0x20
3717
3718 vncipher $out0,$out0,v25
3719 vncipher $out1,$out1,v25
3720 vncipher $out2,$out2,v25
3721 vncipher $out3,$out3,v25
3722 vncipher $out4,$out4,v25
3723 lvx v25,$x10,$key_ # round[4]
3724 bdnz _aesp8_xts_dec5x
3725
3726 subi r0,$taillen,1
3727 vncipher $out0,$out0,v24
3728 vncipher $out1,$out1,v24
3729 vncipher $out2,$out2,v24
3730 vncipher $out3,$out3,v24
3731 vncipher $out4,$out4,v24
3732
3733 andi. r0,r0,16
3734 cmpwi $taillen,0
3735 vncipher $out0,$out0,v25
3736 vncipher $out1,$out1,v25
3737 vncipher $out2,$out2,v25
3738 vncipher $out3,$out3,v25
3739 vncipher $out4,$out4,v25
3740 vxor $twk0,$twk0,v31
3741
3742 sub $inp,$inp,r0
3743 vncipher $out0,$out0,v26
3744 vncipher $out1,$out1,v26
3745 vncipher $out2,$out2,v26
3746 vncipher $out3,$out3,v26
3747 vncipher $out4,$out4,v26
3748 vxor $in1,$twk1,v31
3749
3750 vncipher $out0,$out0,v27
3751 lvx_u $in0,0,$inp
3752 vncipher $out1,$out1,v27
3753 vncipher $out2,$out2,v27
3754 vncipher $out3,$out3,v27
3755 vncipher $out4,$out4,v27
3756 vxor $in2,$twk2,v31
3757
3758 addi $key_,$sp,$FRAME+15 # rewind $key_
3759 vncipher $out0,$out0,v28
3760 vncipher $out1,$out1,v28
3761 vncipher $out2,$out2,v28
3762 vncipher $out3,$out3,v28
3763 vncipher $out4,$out4,v28
3764 lvx v24,$x00,$key_ # re-pre-load round[1]
3765 vxor $in3,$twk3,v31
3766
3767 vncipher $out0,$out0,v29
3768 le?vperm $in0,$in0,$in0,$leperm
3769 vncipher $out1,$out1,v29
3770 vncipher $out2,$out2,v29
3771 vncipher $out3,$out3,v29
3772 vncipher $out4,$out4,v29
3773 lvx v25,$x10,$key_ # re-pre-load round[2]
3774 vxor $in4,$twk4,v31
3775
3776 vncipher $out0,$out0,v30
3777 vncipher $out1,$out1,v30
3778 vncipher $out2,$out2,v30
3779 vncipher $out3,$out3,v30
3780 vncipher $out4,$out4,v30
3781
3782 vncipherlast $out0,$out0,$twk0
3783 vncipherlast $out1,$out1,$in1
3784 vncipherlast $out2,$out2,$in2
3785 vncipherlast $out3,$out3,$in3
3786 vncipherlast $out4,$out4,$in4
3787 mtctr $rounds
3788 blr
3789 .long 0
3790 .byte 0,12,0x14,0,0,0,0,0
3791 ___
3792 }} }}}
3793
3794 my $consts=1;
3795 foreach(split("\n",$code)) {
3796 s/\`([^\`]*)\`/eval($1)/geo;
3797
3798 # constants table endian-specific conversion
3799 if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) {
3800 my $conv=$3;
3801 my @bytes=();
3802
3803 # convert to endian-agnostic format
3804 if ($1 eq "long") {
3805 foreach (split(/,\s*/,$2)) {
3806 my $l = /^0/?oct:int;
3807 push @bytes,($l>>24)&0xff,($l>>16)&0xff,($l>>8)&0xff,$l&0xff;
3808 }
3809 } else {
3810 @bytes = map(/^0/?oct:int,split(/,\s*/,$2));
3811 }
3812
3813 # little-endian conversion
3814 if ($flavour =~ /le$/o) {
3815 SWITCH: for($conv) {
3816 /\?inv/ && do { @bytes=map($_^0xf,@bytes); last; };
3817 /\?rev/ && do { @bytes=reverse(@bytes); last; };
3818 }
3819 }
3820
3821 #emit
3822 print ".byte\t",join(',',map (sprintf("0x%02x",$_),@bytes)),"\n";
3823 next;
3824 }
3825 $consts=0 if (m/Lconsts:/o); # end of table
3826
3827 # instructions prefixed with '?' are endian-specific and need
3828 # to be adjusted accordingly...
3829 if ($flavour =~ /le$/o) { # little-endian
3830 s/le\?//o or
3831 s/be\?/#be#/o or
3832 s/\?lvsr/lvsl/o or
3833 s/\?lvsl/lvsr/o or
3834 s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or
3835 s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or
3836 s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o;
3837 } else { # big-endian
3838 s/le\?/#le#/o or
3839 s/be\?//o or
3840 s/\?([a-z]+)/$1/o;
3841 }
3842
3843 print $_,"\n";
3844 }
3845
3846 close STDOUT;
Linux with added FPC-III support