?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 5.7.7
[linux/fpc-iii.git] / arch / arm / crypto / sha256-armv4.pl
bloba03cf4dfb7818d1275ee0d06ecf633b149adf1cd
1 #!/usr/bin/env perl
2 # SPDX-License-Identifier: GPL-2.0
3
4 # This code is taken from the OpenSSL project but the author (Andy Polyakov)
5 # has relicensed it under the GPLv2. Therefore this program is free software;
6 # you can redistribute it and/or modify it under the terms of the GNU General
7 # Public License version 2 as published by the Free Software Foundation.
8 #
9 # The original headers, including the original license headers, are
10 # included below for completeness.
11
12 # ====================================================================
13 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
14 # project. The module is, however, dual licensed under OpenSSL and
15 # CRYPTOGAMS licenses depending on where you obtain it. For further
16 # details see http://www.openssl.org/~appro/cryptogams/.
17 # ====================================================================
18
19 # SHA256 block procedure for ARMv4. May 2007.
20
21 # Performance is ~2x better than gcc 3.4 generated code and in "abso-
22 # lute" terms is ~2250 cycles per 64-byte block or ~35 cycles per
23 # byte [on single-issue Xscale PXA250 core].
24
25 # July 2010.
26 #
27 # Rescheduling for dual-issue pipeline resulted in 22% improvement on
28 # Cortex A8 core and ~20 cycles per processed byte.
29
30 # February 2011.
31 #
32 # Profiler-assisted and platform-specific optimization resulted in 16%
33 # improvement on Cortex A8 core and ~15.4 cycles per processed byte.
34
35 # September 2013.
36 #
37 # Add NEON implementation. On Cortex A8 it was measured to process one
38 # byte in 12.5 cycles or 23% faster than integer-only code. Snapdragon
39 # S4 does it in 12.5 cycles too, but it's 50% faster than integer-only
40 # code (meaning that latter performs sub-optimally, nothing was done
41 # about it).
42
43 # May 2014.
44 #
45 # Add ARMv8 code path performing at 2.0 cpb on Apple A7.
46
47 while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
48 open STDOUT,">$output";
49
50 $ctx="r0"; $t0="r0";
51 $inp="r1"; $t4="r1";
52 $len="r2"; $t1="r2";
53 $T1="r3"; $t3="r3";
54 $A="r4";
55 $B="r5";
56 $C="r6";
57 $D="r7";
58 $E="r8";
59 $F="r9";
60 $G="r10";
61 $H="r11";
62 @V=($A,$B,$C,$D,$E,$F,$G,$H);
63 $t2="r12";
64 $Ktbl="r14";
65
66 @Sigma0=( 2,13,22);
67 @Sigma1=( 6,11,25);
68 @sigma0=( 7,18, 3);
69 @sigma1=(17,19,10);
70
71 sub BODY_00_15 {
72 my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
73
74 $code.=<<___ if ($i<16);
75 #if __ARM_ARCH__>=7
76 @ ldr $t1,[$inp],#4 @ $i
77 # if $i==15
78 str $inp,[sp,#17*4] @ make room for $t4
79 # endif
80 eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
81 add $a,$a,$t2 @ h+=Maj(a,b,c) from the past
82 eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e)
83 # ifndef __ARMEB__
84 rev $t1,$t1
85 # endif
86 #else
87 @ ldrb $t1,[$inp,#3] @ $i
88 add $a,$a,$t2 @ h+=Maj(a,b,c) from the past
89 ldrb $t2,[$inp,#2]
90 ldrb $t0,[$inp,#1]
91 orr $t1,$t1,$t2,lsl#8
92 ldrb $t2,[$inp],#4
93 orr $t1,$t1,$t0,lsl#16
94 # if $i==15
95 str $inp,[sp,#17*4] @ make room for $t4
96 # endif
97 eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
98 orr $t1,$t1,$t2,lsl#24
99 eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e)
100 #endif
101 ___
102 $code.=<<___;
103 ldr $t2,[$Ktbl],#4 @ *K256++
104 add $h,$h,$t1 @ h+=X[i]
105 str $t1,[sp,#`$i%16`*4]
106 eor $t1,$f,$g
107 add $h,$h,$t0,ror#$Sigma1[0] @ h+=Sigma1(e)
108 and $t1,$t1,$e
109 add $h,$h,$t2 @ h+=K256[i]
110 eor $t1,$t1,$g @ Ch(e,f,g)
111 eor $t0,$a,$a,ror#`$Sigma0[1]-$Sigma0[0]`
112 add $h,$h,$t1 @ h+=Ch(e,f,g)
113 #if $i==31
114 and $t2,$t2,#0xff
115 cmp $t2,#0xf2 @ done?
116 #endif
117 #if $i<15
118 # if __ARM_ARCH__>=7
119 ldr $t1,[$inp],#4 @ prefetch
120 # else
121 ldrb $t1,[$inp,#3]
122 # endif
123 eor $t2,$a,$b @ a^b, b^c in next round
124 #else
125 ldr $t1,[sp,#`($i+2)%16`*4] @ from future BODY_16_xx
126 eor $t2,$a,$b @ a^b, b^c in next round
127 ldr $t4,[sp,#`($i+15)%16`*4] @ from future BODY_16_xx
128 #endif
129 eor $t0,$t0,$a,ror#`$Sigma0[2]-$Sigma0[0]` @ Sigma0(a)
130 and $t3,$t3,$t2 @ (b^c)&=(a^b)
131 add $d,$d,$h @ d+=h
132 eor $t3,$t3,$b @ Maj(a,b,c)
133 add $h,$h,$t0,ror#$Sigma0[0] @ h+=Sigma0(a)
134 @ add $h,$h,$t3 @ h+=Maj(a,b,c)
135 ___
136 ($t2,$t3)=($t3,$t2);
137 }
138
139 sub BODY_16_XX {
140 my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
141
142 $code.=<<___;
143 @ ldr $t1,[sp,#`($i+1)%16`*4] @ $i
144 @ ldr $t4,[sp,#`($i+14)%16`*4]
145 mov $t0,$t1,ror#$sigma0[0]
146 add $a,$a,$t2 @ h+=Maj(a,b,c) from the past
147 mov $t2,$t4,ror#$sigma1[0]
148 eor $t0,$t0,$t1,ror#$sigma0[1]
149 eor $t2,$t2,$t4,ror#$sigma1[1]
150 eor $t0,$t0,$t1,lsr#$sigma0[2] @ sigma0(X[i+1])
151 ldr $t1,[sp,#`($i+0)%16`*4]
152 eor $t2,$t2,$t4,lsr#$sigma1[2] @ sigma1(X[i+14])
153 ldr $t4,[sp,#`($i+9)%16`*4]
154
155 add $t2,$t2,$t0
156 eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]` @ from BODY_00_15
157 add $t1,$t1,$t2
158 eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e)
159 add $t1,$t1,$t4 @ X[i]
160 ___
161 &BODY_00_15(@_);
162 }
163
164 $code=<<___;
165 #ifndef __KERNEL__
166 # include "arm_arch.h"
167 #else
168 # define __ARM_ARCH__ __LINUX_ARM_ARCH__
169 # define __ARM_MAX_ARCH__ 7
170 #endif
171
172 .text
173 #if __ARM_ARCH__<7
174 .code 32
175 #else
176 .syntax unified
177 # ifdef __thumb2__
178 # define adrl adr
179 .thumb
180 # else
181 .code 32
182 # endif
183 #endif
184
185 .type K256,%object
186 .align 5
187 K256:
188 .word 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
189 .word 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
190 .word 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
191 .word 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
192 .word 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
193 .word 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
194 .word 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
195 .word 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
196 .word 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
197 .word 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
198 .word 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
199 .word 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
200 .word 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
201 .word 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
202 .word 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
203 .word 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
204 .size K256,.-K256
205 .word 0 @ terminator
206 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
207 .LOPENSSL_armcap:
208 .word OPENSSL_armcap_P-sha256_block_data_order
209 #endif
210 .align 5
211
212 .global sha256_block_data_order
213 .type sha256_block_data_order,%function
214 sha256_block_data_order:
215 .Lsha256_block_data_order:
216 #if __ARM_ARCH__<7
217 sub r3,pc,#8 @ sha256_block_data_order
218 #else
219 adr r3,.Lsha256_block_data_order
220 #endif
221 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
222 ldr r12,.LOPENSSL_armcap
223 ldr r12,[r3,r12] @ OPENSSL_armcap_P
224 tst r12,#ARMV8_SHA256
225 bne .LARMv8
226 tst r12,#ARMV7_NEON
227 bne .LNEON
228 #endif
229 add $len,$inp,$len,lsl#6 @ len to point at the end of inp
230 stmdb sp!,{$ctx,$inp,$len,r4-r11,lr}
231 ldmia $ctx,{$A,$B,$C,$D,$E,$F,$G,$H}
232 sub $Ktbl,r3,#256+32 @ K256
233 sub sp,sp,#16*4 @ alloca(X[16])
234 .Loop:
235 # if __ARM_ARCH__>=7
236 ldr $t1,[$inp],#4
237 # else
238 ldrb $t1,[$inp,#3]
239 # endif
240 eor $t3,$B,$C @ magic
241 eor $t2,$t2,$t2
242 ___
243 for($i=0;$i<16;$i++) { &BODY_00_15($i,@V); unshift(@V,pop(@V)); }
244 $code.=".Lrounds_16_xx:\n";
245 for (;$i<32;$i++) { &BODY_16_XX($i,@V); unshift(@V,pop(@V)); }
246 $code.=<<___;
247 #if __ARM_ARCH__>=7
248 ite eq @ Thumb2 thing, sanity check in ARM
249 #endif
250 ldreq $t3,[sp,#16*4] @ pull ctx
251 bne .Lrounds_16_xx
252
253 add $A,$A,$t2 @ h+=Maj(a,b,c) from the past
254 ldr $t0,[$t3,#0]
255 ldr $t1,[$t3,#4]
256 ldr $t2,[$t3,#8]
257 add $A,$A,$t0
258 ldr $t0,[$t3,#12]
259 add $B,$B,$t1
260 ldr $t1,[$t3,#16]
261 add $C,$C,$t2
262 ldr $t2,[$t3,#20]
263 add $D,$D,$t0
264 ldr $t0,[$t3,#24]
265 add $E,$E,$t1
266 ldr $t1,[$t3,#28]
267 add $F,$F,$t2
268 ldr $inp,[sp,#17*4] @ pull inp
269 ldr $t2,[sp,#18*4] @ pull inp+len
270 add $G,$G,$t0
271 add $H,$H,$t1
272 stmia $t3,{$A,$B,$C,$D,$E,$F,$G,$H}
273 cmp $inp,$t2
274 sub $Ktbl,$Ktbl,#256 @ rewind Ktbl
275 bne .Loop
276
277 add sp,sp,#`16+3`*4 @ destroy frame
278 #if __ARM_ARCH__>=5
279 ldmia sp!,{r4-r11,pc}
280 #else
281 ldmia sp!,{r4-r11,lr}
282 tst lr,#1
283 moveq pc,lr @ be binary compatible with V4, yet
284 bx lr @ interoperable with Thumb ISA:-)
285 #endif
286 .size sha256_block_data_order,.-sha256_block_data_order
287 ___
288 ######################################################################
289 # NEON stuff
290 #
291 {{{
292 my @X=map("q$_",(0..3));
293 my ($T0,$T1,$T2,$T3,$T4,$T5)=("q8","q9","q10","q11","d24","d25");
294 my $Xfer=$t4;
295 my $j=0;
296
297 sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; }
298 sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; }
299
300 sub AUTOLOAD() # thunk [simplified] x86-style perlasm
301 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
302 my $arg = pop;
303 $arg = "#$arg" if ($arg*1 eq $arg);
304 $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
305 }
306
307 sub Xupdate()
308 { use integer;
309 my $body = shift;
310 my @insns = (&$body,&$body,&$body,&$body);
311 my ($a,$b,$c,$d,$e,$f,$g,$h);
312
313 &vext_8 ($T0,@X[0],@X[1],4); # X[1..4]
314 eval(shift(@insns));
315 eval(shift(@insns));
316 eval(shift(@insns));
317 &vext_8 ($T1,@X[2],@X[3],4); # X[9..12]
318 eval(shift(@insns));
319 eval(shift(@insns));
320 eval(shift(@insns));
321 &vshr_u32 ($T2,$T0,$sigma0[0]);
322 eval(shift(@insns));
323 eval(shift(@insns));
324 &vadd_i32 (@X[0],@X[0],$T1); # X[0..3] += X[9..12]
325 eval(shift(@insns));
326 eval(shift(@insns));
327 &vshr_u32 ($T1,$T0,$sigma0[2]);
328 eval(shift(@insns));
329 eval(shift(@insns));
330 &vsli_32 ($T2,$T0,32-$sigma0[0]);
331 eval(shift(@insns));
332 eval(shift(@insns));
333 &vshr_u32 ($T3,$T0,$sigma0[1]);
334 eval(shift(@insns));
335 eval(shift(@insns));
336 &veor ($T1,$T1,$T2);
337 eval(shift(@insns));
338 eval(shift(@insns));
339 &vsli_32 ($T3,$T0,32-$sigma0[1]);
340 eval(shift(@insns));
341 eval(shift(@insns));
342 &vshr_u32 ($T4,&Dhi(@X[3]),$sigma1[0]);
343 eval(shift(@insns));
344 eval(shift(@insns));
345 &veor ($T1,$T1,$T3); # sigma0(X[1..4])
346 eval(shift(@insns));
347 eval(shift(@insns));
348 &vsli_32 ($T4,&Dhi(@X[3]),32-$sigma1[0]);
349 eval(shift(@insns));
350 eval(shift(@insns));
351 &vshr_u32 ($T5,&Dhi(@X[3]),$sigma1[2]);
352 eval(shift(@insns));
353 eval(shift(@insns));
354 &vadd_i32 (@X[0],@X[0],$T1); # X[0..3] += sigma0(X[1..4])
355 eval(shift(@insns));
356 eval(shift(@insns));
357 &veor ($T5,$T5,$T4);
358 eval(shift(@insns));
359 eval(shift(@insns));
360 &vshr_u32 ($T4,&Dhi(@X[3]),$sigma1[1]);
361 eval(shift(@insns));
362 eval(shift(@insns));
363 &vsli_32 ($T4,&Dhi(@X[3]),32-$sigma1[1]);
364 eval(shift(@insns));
365 eval(shift(@insns));
366 &veor ($T5,$T5,$T4); # sigma1(X[14..15])
367 eval(shift(@insns));
368 eval(shift(@insns));
369 &vadd_i32 (&Dlo(@X[0]),&Dlo(@X[0]),$T5);# X[0..1] += sigma1(X[14..15])
370 eval(shift(@insns));
371 eval(shift(@insns));
372 &vshr_u32 ($T4,&Dlo(@X[0]),$sigma1[0]);
373 eval(shift(@insns));
374 eval(shift(@insns));
375 &vsli_32 ($T4,&Dlo(@X[0]),32-$sigma1[0]);
376 eval(shift(@insns));
377 eval(shift(@insns));
378 &vshr_u32 ($T5,&Dlo(@X[0]),$sigma1[2]);
379 eval(shift(@insns));
380 eval(shift(@insns));
381 &veor ($T5,$T5,$T4);
382 eval(shift(@insns));
383 eval(shift(@insns));
384 &vshr_u32 ($T4,&Dlo(@X[0]),$sigma1[1]);
385 eval(shift(@insns));
386 eval(shift(@insns));
387 &vld1_32 ("{$T0}","[$Ktbl,:128]!");
388 eval(shift(@insns));
389 eval(shift(@insns));
390 &vsli_32 ($T4,&Dlo(@X[0]),32-$sigma1[1]);
391 eval(shift(@insns));
392 eval(shift(@insns));
393 &veor ($T5,$T5,$T4); # sigma1(X[16..17])
394 eval(shift(@insns));
395 eval(shift(@insns));
396 &vadd_i32 (&Dhi(@X[0]),&Dhi(@X[0]),$T5);# X[2..3] += sigma1(X[16..17])
397 eval(shift(@insns));
398 eval(shift(@insns));
399 &vadd_i32 ($T0,$T0,@X[0]);
400 while($#insns>=2) { eval(shift(@insns)); }
401 &vst1_32 ("{$T0}","[$Xfer,:128]!");
402 eval(shift(@insns));
403 eval(shift(@insns));
404
405 push(@X,shift(@X)); # "rotate" X[]
406 }
407
408 sub Xpreload()
409 { use integer;
410 my $body = shift;
411 my @insns = (&$body,&$body,&$body,&$body);
412 my ($a,$b,$c,$d,$e,$f,$g,$h);
413
414 eval(shift(@insns));
415 eval(shift(@insns));
416 eval(shift(@insns));
417 eval(shift(@insns));
418 &vld1_32 ("{$T0}","[$Ktbl,:128]!");
419 eval(shift(@insns));
420 eval(shift(@insns));
421 eval(shift(@insns));
422 eval(shift(@insns));
423 &vrev32_8 (@X[0],@X[0]);
424 eval(shift(@insns));
425 eval(shift(@insns));
426 eval(shift(@insns));
427 eval(shift(@insns));
428 &vadd_i32 ($T0,$T0,@X[0]);
429 foreach (@insns) { eval; } # remaining instructions
430 &vst1_32 ("{$T0}","[$Xfer,:128]!");
431
432 push(@X,shift(@X)); # "rotate" X[]
433 }
434
435 sub body_00_15 () {
436 (
437 '($a,$b,$c,$d,$e,$f,$g,$h)=@V;'.
438 '&add ($h,$h,$t1)', # h+=X[i]+K[i]
439 '&eor ($t1,$f,$g)',
440 '&eor ($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))',
441 '&add ($a,$a,$t2)', # h+=Maj(a,b,c) from the past
442 '&and ($t1,$t1,$e)',
443 '&eor ($t2,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))', # Sigma1(e)
444 '&eor ($t0,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))',
445 '&eor ($t1,$t1,$g)', # Ch(e,f,g)
446 '&add ($h,$h,$t2,"ror#$Sigma1[0]")', # h+=Sigma1(e)
447 '&eor ($t2,$a,$b)', # a^b, b^c in next round
448 '&eor ($t0,$t0,$a,"ror#".($Sigma0[2]-$Sigma0[0]))', # Sigma0(a)
449 '&add ($h,$h,$t1)', # h+=Ch(e,f,g)
450 '&ldr ($t1,sprintf "[sp,#%d]",4*(($j+1)&15)) if (($j&15)!=15);'.
451 '&ldr ($t1,"[$Ktbl]") if ($j==15);'.
452 '&ldr ($t1,"[sp,#64]") if ($j==31)',
453 '&and ($t3,$t3,$t2)', # (b^c)&=(a^b)
454 '&add ($d,$d,$h)', # d+=h
455 '&add ($h,$h,$t0,"ror#$Sigma0[0]");'. # h+=Sigma0(a)
456 '&eor ($t3,$t3,$b)', # Maj(a,b,c)
457 '$j++; unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);'
458 )
459 }
460
461 $code.=<<___;
462 #if __ARM_MAX_ARCH__>=7
463 .arch armv7-a
464 .fpu neon
465
466 .global sha256_block_data_order_neon
467 .type sha256_block_data_order_neon,%function
468 .align 4
469 sha256_block_data_order_neon:
470 .LNEON:
471 stmdb sp!,{r4-r12,lr}
472
473 sub $H,sp,#16*4+16
474 adrl $Ktbl,K256
475 bic $H,$H,#15 @ align for 128-bit stores
476 mov $t2,sp
477 mov sp,$H @ alloca
478 add $len,$inp,$len,lsl#6 @ len to point at the end of inp
479
480 vld1.8 {@X[0]},[$inp]!
481 vld1.8 {@X[1]},[$inp]!
482 vld1.8 {@X[2]},[$inp]!
483 vld1.8 {@X[3]},[$inp]!
484 vld1.32 {$T0},[$Ktbl,:128]!
485 vld1.32 {$T1},[$Ktbl,:128]!
486 vld1.32 {$T2},[$Ktbl,:128]!
487 vld1.32 {$T3},[$Ktbl,:128]!
488 vrev32.8 @X[0],@X[0] @ yes, even on
489 str $ctx,[sp,#64]
490 vrev32.8 @X[1],@X[1] @ big-endian
491 str $inp,[sp,#68]
492 mov $Xfer,sp
493 vrev32.8 @X[2],@X[2]
494 str $len,[sp,#72]
495 vrev32.8 @X[3],@X[3]
496 str $t2,[sp,#76] @ save original sp
497 vadd.i32 $T0,$T0,@X[0]
498 vadd.i32 $T1,$T1,@X[1]
499 vst1.32 {$T0},[$Xfer,:128]!
500 vadd.i32 $T2,$T2,@X[2]
501 vst1.32 {$T1},[$Xfer,:128]!
502 vadd.i32 $T3,$T3,@X[3]
503 vst1.32 {$T2},[$Xfer,:128]!
504 vst1.32 {$T3},[$Xfer,:128]!
505
506 ldmia $ctx,{$A-$H}
507 sub $Xfer,$Xfer,#64
508 ldr $t1,[sp,#0]
509 eor $t2,$t2,$t2
510 eor $t3,$B,$C
511 b .L_00_48
512
513 .align 4
514 .L_00_48:
515 ___
516 &Xupdate(\&body_00_15);
517 &Xupdate(\&body_00_15);
518 &Xupdate(\&body_00_15);
519 &Xupdate(\&body_00_15);
520 $code.=<<___;
521 teq $t1,#0 @ check for K256 terminator
522 ldr $t1,[sp,#0]
523 sub $Xfer,$Xfer,#64
524 bne .L_00_48
525
526 ldr $inp,[sp,#68]
527 ldr $t0,[sp,#72]
528 sub $Ktbl,$Ktbl,#256 @ rewind $Ktbl
529 teq $inp,$t0
530 it eq
531 subeq $inp,$inp,#64 @ avoid SEGV
532 vld1.8 {@X[0]},[$inp]! @ load next input block
533 vld1.8 {@X[1]},[$inp]!
534 vld1.8 {@X[2]},[$inp]!
535 vld1.8 {@X[3]},[$inp]!
536 it ne
537 strne $inp,[sp,#68]
538 mov $Xfer,sp
539 ___
540 &Xpreload(\&body_00_15);
541 &Xpreload(\&body_00_15);
542 &Xpreload(\&body_00_15);
543 &Xpreload(\&body_00_15);
544 $code.=<<___;
545 ldr $t0,[$t1,#0]
546 add $A,$A,$t2 @ h+=Maj(a,b,c) from the past
547 ldr $t2,[$t1,#4]
548 ldr $t3,[$t1,#8]
549 ldr $t4,[$t1,#12]
550 add $A,$A,$t0 @ accumulate
551 ldr $t0,[$t1,#16]
552 add $B,$B,$t2
553 ldr $t2,[$t1,#20]
554 add $C,$C,$t3
555 ldr $t3,[$t1,#24]
556 add $D,$D,$t4
557 ldr $t4,[$t1,#28]
558 add $E,$E,$t0
559 str $A,[$t1],#4
560 add $F,$F,$t2
561 str $B,[$t1],#4
562 add $G,$G,$t3
563 str $C,[$t1],#4
564 add $H,$H,$t4
565 str $D,[$t1],#4
566 stmia $t1,{$E-$H}
567
568 ittte ne
569 movne $Xfer,sp
570 ldrne $t1,[sp,#0]
571 eorne $t2,$t2,$t2
572 ldreq sp,[sp,#76] @ restore original sp
573 itt ne
574 eorne $t3,$B,$C
575 bne .L_00_48
576
577 ldmia sp!,{r4-r12,pc}
578 .size sha256_block_data_order_neon,.-sha256_block_data_order_neon
579 #endif
580 ___
581 }}}
582 ######################################################################
583 # ARMv8 stuff
584 #
585 {{{
586 my ($ABCD,$EFGH,$abcd)=map("q$_",(0..2));
587 my @MSG=map("q$_",(8..11));
588 my ($W0,$W1,$ABCD_SAVE,$EFGH_SAVE)=map("q$_",(12..15));
589 my $Ktbl="r3";
590
591 $code.=<<___;
592 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
593
594 # ifdef __thumb2__
595 # define INST(a,b,c,d) .byte c,d|0xc,a,b
596 # else
597 # define INST(a,b,c,d) .byte a,b,c,d
598 # endif
599
600 .type sha256_block_data_order_armv8,%function
601 .align 5
602 sha256_block_data_order_armv8:
603 .LARMv8:
604 vld1.32 {$ABCD,$EFGH},[$ctx]
605 # ifdef __thumb2__
606 adr $Ktbl,.LARMv8
607 sub $Ktbl,$Ktbl,#.LARMv8-K256
608 # else
609 adrl $Ktbl,K256
610 # endif
611 add $len,$inp,$len,lsl#6 @ len to point at the end of inp
612
613 .Loop_v8:
614 vld1.8 {@MSG[0]-@MSG[1]},[$inp]!
615 vld1.8 {@MSG[2]-@MSG[3]},[$inp]!
616 vld1.32 {$W0},[$Ktbl]!
617 vrev32.8 @MSG[0],@MSG[0]
618 vrev32.8 @MSG[1],@MSG[1]
619 vrev32.8 @MSG[2],@MSG[2]
620 vrev32.8 @MSG[3],@MSG[3]
621 vmov $ABCD_SAVE,$ABCD @ offload
622 vmov $EFGH_SAVE,$EFGH
623 teq $inp,$len
624 ___
625 for($i=0;$i<12;$i++) {
626 $code.=<<___;
627 vld1.32 {$W1},[$Ktbl]!
628 vadd.i32 $W0,$W0,@MSG[0]
629 sha256su0 @MSG[0],@MSG[1]
630 vmov $abcd,$ABCD
631 sha256h $ABCD,$EFGH,$W0
632 sha256h2 $EFGH,$abcd,$W0
633 sha256su1 @MSG[0],@MSG[2],@MSG[3]
634 ___
635 ($W0,$W1)=($W1,$W0); push(@MSG,shift(@MSG));
636 }
637 $code.=<<___;
638 vld1.32 {$W1},[$Ktbl]!
639 vadd.i32 $W0,$W0,@MSG[0]
640 vmov $abcd,$ABCD
641 sha256h $ABCD,$EFGH,$W0
642 sha256h2 $EFGH,$abcd,$W0
643
644 vld1.32 {$W0},[$Ktbl]!
645 vadd.i32 $W1,$W1,@MSG[1]
646 vmov $abcd,$ABCD
647 sha256h $ABCD,$EFGH,$W1
648 sha256h2 $EFGH,$abcd,$W1
649
650 vld1.32 {$W1},[$Ktbl]
651 vadd.i32 $W0,$W0,@MSG[2]
652 sub $Ktbl,$Ktbl,#256-16 @ rewind
653 vmov $abcd,$ABCD
654 sha256h $ABCD,$EFGH,$W0
655 sha256h2 $EFGH,$abcd,$W0
656
657 vadd.i32 $W1,$W1,@MSG[3]
658 vmov $abcd,$ABCD
659 sha256h $ABCD,$EFGH,$W1
660 sha256h2 $EFGH,$abcd,$W1
661
662 vadd.i32 $ABCD,$ABCD,$ABCD_SAVE
663 vadd.i32 $EFGH,$EFGH,$EFGH_SAVE
664 it ne
665 bne .Loop_v8
666
667 vst1.32 {$ABCD,$EFGH},[$ctx]
668
669 ret @ bx lr
670 .size sha256_block_data_order_armv8,.-sha256_block_data_order_armv8
671 #endif
672 ___
673 }}}
674 $code.=<<___;
675 .asciz "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
676 .align 2
677 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
678 .comm OPENSSL_armcap_P,4,4
679 #endif
680 ___
681
682 open SELF,$0;
683 while(<SELF>) {
684 next if (/^#!/);
685 last if (!s/^#/@/ and !/^$/);
686 print;
687 }
688 close SELF;
689
690 { my %opcode = (
691 "sha256h" => 0xf3000c40, "sha256h2" => 0xf3100c40,
692 "sha256su0" => 0xf3ba03c0, "sha256su1" => 0xf3200c40 );
693
694 sub unsha256 {
695 my ($mnemonic,$arg)=@_;
696
697 if ($arg =~ m/q([0-9]+)(?:,\s*q([0-9]+))?,\s*q([0-9]+)/o) {
698 my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19)
699 |(($2&7)<<17)|(($2&8)<<4)
700 |(($3&7)<<1) |(($3&8)<<2);
701 # since ARMv7 instructions are always encoded little-endian.
702 # correct solution is to use .inst directive, but older
703 # assemblers don't implement it:-(
704 sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s",
705 $word&0xff,($word>>8)&0xff,
706 ($word>>16)&0xff,($word>>24)&0xff,
707 $mnemonic,$arg;
708 }
709 }
710 }
711
712 foreach (split($/,$code)) {
713
714 s/\`([^\`]*)\`/eval $1/geo;
715
716 s/\b(sha256\w+)\s+(q.*)/unsha256($1,$2)/geo;
717
718 s/\bret\b/bx lr/go or
719 s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4
720
721 print $_,"\n";
722 }
723
724 close STDOUT; # enforce flush
Linux with added FPC-III support