3 # ====================================================================
4 # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
12 # Provided that UltraSPARC VIS instructions are pipe-lined(*) and
13 # pairable(*) with IALU ones, offloading of Xupdate to the UltraSPARC
14 # Graphic Unit would make it possible to achieve higher instruction-
15 # level parallelism, ILP, and thus higher performance. It should be
16 # explicitly noted that ILP is the keyword, and it means that this
17 # code would be unsuitable for cores like UltraSPARC-Tx. The idea is
18 # not really novel, Sun had VIS-powered implementation for a while.
19 # Unlike Sun's implementation this one can process multiple unaligned
20 # input blocks, and as such works as drop-in replacement for OpenSSL
21 # sha1_block_data_order. Performance improvement was measured to be
22 # 40% over pure IALU sha1-sparcv9.pl on UltraSPARC-IIi, but 12% on
23 # UltraSPARC-III. See below for discussion...
25 # The module does not present direct interest for OpenSSL, because
26 # it doesn't provide better performance on contemporary SPARCv9 CPUs,
27 # UltraSPARC-Tx and SPARC64-V[II] to be specific. Those who feel they
28 # absolutely must score on UltraSPARC-I-IV can simply replace
29 # crypto/sha/asm/sha1-sparcv9.pl with this module.
31 # (*) "Pipe-lined" means that even if it takes several cycles to
32 # complete, next instruction using same functional unit [but not
33 # depending on the result of the current instruction] can start
34 # execution without having to wait for the unit. "Pairable"
35 # means that two [or more] independent instructions can be
36 # issued at the very same time.
39 for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); }
40 if ($bits==64) { $bias=2047; $frame=192; }
41 else { $bias=0; $frame=112; }
44 open STDOUT
,">$output";
78 @VK=($VK_00_19,$VK_20_39,$VK_40_59,$VK_60_79);
79 @X=("%f0", "%f1", "%f2", "%f3", "%f4", "%f5", "%f6", "%f7",
80 "%f8", "%f9","%f10","%f11","%f12","%f13","%f14","%f15","%f16");
82 # This is reference 2x-parallelized VIS-powered Xupdate procedure. It
83 # covers even K_NN_MM addition...
86 my $K=@VK[($i+16)/20];
89 # [ provided that GSR.alignaddr_offset is 5, $mul contains
90 # 0x100ULL<<32|0x100 value and K_NN_MM are pre-loaded to
91 # chosen registers... ]
93 fxors
@X[($j+13)%16],@X[$j],@X[$j] !-1/-1/-1:X
[0]^=X
[13]
94 fxors
@X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X
[1]^=X
[14]
95 fxor
@X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp
=X
[2,3]^X
[8,9]
96 fxor
%f18,@X[$j],@X[$j] ! 2/ 4/ 3:X
[0,1]^=X
[2,3]^X
[8,9]
97 faligndata
@X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp
=X
[0,1]>>>24
98 fpadd32
@X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X
[0,1]<<=1
99 fmul8ulx16
%f18,$fmul,%f18 ! 5/10/ 7:Tmp
>>=7, Tmp
&=1
100 ![fxors
%f15,%f2,%f2]
101 for %f18,@X[$j],@X[$j] ! 8/14/10:X
[0,1]|=Tmp
102 ![fxors
%f0,%f3,%f3] !10/17/12:X
[0] dependency
103 fpadd32
$K,@X[$j],%f20
104 std
%f20,[$Xfer+`4*$j`]
106 # The numbers delimited with slash are the earliest possible dispatch
107 # cycles for given instruction assuming 1 cycle latency for simple VIS
108 # instructions, such as on UltraSPARC-I&II, 3 cycles latency, such as
109 # on UltraSPARC-III&IV, and 2 cycles latency(*), respectively. Being
110 # 2x-parallelized the procedure is "worth" 5, 8.5 or 6 ticks per SHA1
111 # round. As [long as] FPU/VIS instructions are perfectly pairable with
112 # IALU ones, the round timing is defined by the maximum between VIS
113 # and IALU timings. The latter varies from round to round and averages
114 # out at 6.25 ticks. This means that USI&II should operate at IALU
115 # rate, while USIII&IV - at VIS rate. This explains why performance
116 # improvement varies among processors. Well, given that pure IALU
117 # sha1-sparcv9.pl module exhibits virtually uniform performance of
118 # ~9.3 cycles per SHA1 round. Timings mentioned above are theoretical
119 # lower limits. Real-life performance was measured to be 6.6 cycles
120 # per SHA1 round on USIIi and 8.3 on USIII. The latter is lower than
121 # half-round VIS timing, because there are 16 Xupdate-free rounds,
122 # which "push down" average theoretical timing to 8 cycles...
124 # (*) SPARC64-V[II] was originally believed to have 2 cycles VIS
125 # latency. Well, it might have, but it doesn't have dedicated
126 # VIS-unit. Instead, VIS instructions are executed by other
127 # functional units, ones used here - by IALU. This doesn't
128 # improve effective ILP...
131 # The reference Xupdate procedure is then "strained" over *pairs* of
132 # BODY_NN_MM and kind of modulo-scheduled in respect to X[n]^=X[n+13]
133 # and K_NN_MM addition. It's "running" 15 rounds ahead, which leaves
134 # plenty of room to amortize for read-after-write hazard, as well as
135 # to fetch and align input for the next spin. The VIS instructions are
136 # scheduled for latency of 2 cycles, because there are not enough IALU
137 # instructions to schedule for latency of 3, while scheduling for 1
138 # would give no gain on USI&II anyway.
141 my ($i,$a,$b,$c,$d,$e)=@_;
143 my $k=($j+16+2)%16; # ahead reference
144 my $l=($j+16-2)%16; # behind reference
145 my $K=@VK[($j+16-2)/20];
149 $code.=<<___
if (!($i&1));
152 ld
[$Xfer+`4*($i%16)`],$Xi
153 fxors
@X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X
[1]^=X
[14]
156 fxor
@X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp
=X
[2,3]^X
[8,9]
161 fxor
%f18,@X[$j],@X[$j] ! 2/ 4/ 3:X
[0,1]^=X
[2,3]^X
[8,9]
166 faligndata
@X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp
=X
[0,1]>>>24
168 $code.=<<___
if ($i&1);
171 ld
[$Xfer+`4*($i%16)`],$Xi
172 fpadd32
@X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X
[0,1]<<=1
175 fmul8ulx16
%f18,$fmul,%f18 ! 5/10/ 7:Tmp
>>=7, Tmp
&=1
178 fpadd32
$K,@X[$l],%f20 !
181 fxors
@X[($k+13)%16],@X[$k],@X[$k] !-1/-1/-1:X
[0]^=X
[13]
184 fxor
%f18,@X[$j],@X[$j] ! 8/14/10:X
[0,1]|=Tmp
188 $code.=<<___
if ($i&1 && $i>=2);
189 std
%f20,[$Xfer+`4*$l`] !
194 my ($i,$a,$b,$c,$d,$e)=@_;
196 my $k=($j+16+2)%16; # ahead reference
197 my $l=($j+16-2)%16; # behind reference
198 my $K=@VK[($j+16-2)/20];
202 $code.=<<___
if (!($i&1) && $i<64);
204 ld
[$Xfer+`4*($i%16)`],$Xi
205 fxors
@X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X
[1]^=X
[14]
208 fxor
@X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp
=X
[2,3]^X
[8,9]
213 fxor
%f18,@X[$j],@X[$j] ! 2/ 4/ 3:X
[0,1]^=X
[2,3]^X
[8,9]
218 faligndata
@X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp
=X
[0,1]>>>24
220 $code.=<<___
if ($i&1 && $i<64);
222 ld
[$Xfer+`4*($i%16)`],$Xi
223 fpadd32
@X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X
[0,1]<<=1
226 fmul8ulx16
%f18,$fmul,%f18 ! 5/10/ 7:Tmp
>>=7, Tmp
&=1
229 fpadd32
$K,@X[$l],%f20 !
232 fxors
@X[($k+13)%16],@X[$k],@X[$k] !-1/-1/-1:X
[0]^=X
[13]
235 fxor
%f18,@X[$j],@X[$j] ! 8/14/10:X
[0,1]|=Tmp
238 std
%f20,[$Xfer+`4*$l`] !
240 $code.=<<___
if ($i==64);
242 ld
[$Xfer+`4*($i%16)`],$Xi
243 fpadd32
$K,@X[$l],%f20
250 std
%f20,[$Xfer+`4*$l`]
256 $code.=<<___
if ($i>64);
258 ld
[$Xfer+`4*($i%16)`],$Xi
273 my ($i,$a,$b,$c,$d,$e)=@_;
275 my $k=($j+16+2)%16; # ahead reference
276 my $l=($j+16-2)%16; # behind reference
277 my $K=@VK[($j+16-2)/20];
281 $code.=<<___
if (!($i&1));
283 ld
[$Xfer+`4*($i%16)`],$Xi
284 fxors
@X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X
[1]^=X
[14]
287 fxor
@X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp
=X
[2,3]^X
[8,9]
292 fxor
%f18,@X[$j],@X[$j] ! 2/ 4/ 3:X
[0,1]^=X
[2,3]^X
[8,9]
297 faligndata
@X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp
=X
[0,1]>>>24
300 fpadd32
@X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X
[0,1]<<=1
302 $code.=<<___
if ($i&1);
304 ld
[$Xfer+`4*($i%16)`],$Xi
307 fmul8ulx16
%f18,$fmul,%f18 ! 5/10/ 7:Tmp
>>=7, Tmp
&=1
310 fpadd32
$K,@X[$l],%f20 !
313 fxors
@X[($k+13)%16],@X[$k],@X[$k] !-1/-1/-1:X
[0]^=X
[13]
316 fxor
%f18,@X[$j],@X[$j] ! 8/14/10:X
[0,1]|=Tmp
321 std
%f20,[$Xfer+`4*$l`] !
325 # If there is more data to process, then we pre-fetch the data for
326 # next iteration in last ten rounds...
328 my ($i,$a,$b,$c,$d,$e)=@_;
334 $code.=<<___
if ($i==70);
336 ld
[$Xfer+`4*($i%16)`],$Xi
351 and $nXfer,255,$nXfer
352 alignaddr
%g0,$align,%g0
353 add
$base,$nXfer,$nXfer
355 $code.=<<___
if ($i==71);
357 ld
[$Xfer+`4*($i%16)`],$Xi
369 $code.=<<___
if ($i>=72);
370 faligndata
@X[$m],@X[$m+2],@X[$m]
372 ld
[$Xfer+`4*($i%16)`],$Xi
377 fpadd32
$VK_00_19,@X[$m],%f20
385 $code.=<<___
if ($i<77);
386 ldd
[$inp+`8*($i+1-70)`],@X[2*($i+1-70)]
388 $code.=<<___
if ($i==77); # redundant if $inp was aligned
391 ldd
[$inp+$tmp0],@X[16]
393 $code.=<<___
if ($i>=72);
394 std
%f20,[$nXfer+`4*$m`]
399 .section
".text",#alloc,#execinstr
403 .long
0x5a827999,0x5a827999 ! K_00_19
404 .long
0x6ed9eba1,0x6ed9eba1 ! K_20_39
405 .long
0x8f1bbcdc,0x8f1bbcdc ! K_40_59
406 .long
0xca62c1d6,0xca62c1d6 ! K_60_79
407 .long
0x00000100,0x00000100
409 .type vis_const
,#object
410 .size vis_const
,(.-vis_const
)
412 .globl sha1_block_data_order
413 sha1_block_data_order
:
415 add
%fp,$bias-256,$base
418 add
%o7,vis_const
-1b
,$tmp0
420 ldd
[$tmp0+0],$VK_00_19
421 ldd
[$tmp0+8],$VK_20_39
422 ldd
[$tmp0+16],$VK_40_59
423 ldd
[$tmp0+24],$VK_60_79
429 sub $base,$bias+$frame,%sp
436 ! X
[16] is maintained
in FP register bank
437 alignaddr
%g0,$align,%g0
445 add
$base,$Xfer,$Xfer
449 brz
,pt
$align,.Laligned
453 faligndata
@X[0],@X[2],@X[0]
454 faligndata
@X[2],@X[4],@X[2]
455 faligndata
@X[4],@X[6],@X[4]
456 faligndata
@X[6],@X[8],@X[6]
457 faligndata
@X[8],@X[10],@X[8]
458 faligndata
@X[10],@X[12],@X[10]
459 faligndata
@X[12],@X[14],@X[12]
460 faligndata
@X[14],@X[16],@X[14]
465 alignaddr
%g0,$tmp0,%g0
466 fpadd32
$VK_00_19,@X[0],%f16
467 fpadd32
$VK_00_19,@X[2],%f18
468 fpadd32
$VK_00_19,@X[4],%f20
469 fpadd32
$VK_00_19,@X[6],%f22
470 fpadd32
$VK_00_19,@X[8],%f24
471 fpadd32
$VK_00_19,@X[10],%f26
472 fpadd32
$VK_00_19,@X[12],%f28
473 fpadd32
$VK_00_19,@X[14],%f30
485 fxors
@X[13],@X[0],@X[0]
492 for ($i=0;$i<20;$i++) { &BODY_00_19
($i,@V); unshift(@V,pop(@V)); }
493 for (;$i<40;$i++) { &BODY_20_39
($i,@V); unshift(@V,pop(@V)); }
494 for (;$i<60;$i++) { &BODY_40_59
($i,@V); unshift(@V,pop(@V)); }
495 for (;$i<70;$i++) { &BODY_20_39
($i,@V); unshift(@V,pop(@V)); }
498 bz
,pn
`$bits==32?"%icc":"%xcc"`,.Ltail
501 for (;$i<80;$i++) { &BODY_70_79
($i,@V); unshift(@V,pop(@V)); }
509 fxors
@X[13],@X[0],@X[0]
515 alignaddr
%g0,$tmp0,%g0
523 for($i=70;$i<80;$i++) { &BODY_20_39
($i,@V); unshift(@V,pop(@V)); }
539 .type sha1_block_data_order
,#function
540 .size sha1_block_data_order
,(.-sha1_block_data_order
)
541 .asciz
"SHA1 block transform for SPARCv9a, CRYPTOGAMS by <appro\@openssl.org>"
545 # Purpose of these subroutines is to explicitly encode VIS instructions,
546 # so that one can compile the module without having to specify VIS
547 # extentions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a.
548 # Idea is to reserve for option to produce "universal" binary and let
549 # programmer detect if current CPU is VIS capable at run-time.
551 my ($mnemonic,$rs1,$rs2,$rd)=@_;
553 my %visopf = ( "fmul8ulx16" => 0x037,
554 "faligndata" => 0x048,
559 $ref = "$mnemonic\t$rs1,$rs2,$rd";
561 if ($opf=$visopf{$mnemonic}) {
562 foreach ($rs1,$rs2,$rd) {
563 return $ref if (!/%f([0-9]{1,2})/);
566 return $ref if ($1&1);
567 # re-encode for upper double register addressing
572 return sprintf ".word\t0x%08x !%s",
573 0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2,
580 my ($mnemonic,$rs1,$rs2,$rd)=@_;
581 my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 );
582 my $ref="$mnemonic\t$rs1,$rs2,$rd";
584 foreach ($rs1,$rs2,$rd) {
585 if (/%([goli])([0-7])/) { $_=$bias{$1}+$2; }
586 else { return $ref; }
588 return sprintf ".word\t0x%08x !%s",
589 0x81b00300|$rd<<25|$rs1<<14|$rs2,
593 $code =~ s/\`([^\`]*)\`/eval $1/gem;
594 $code =~ s
/\b(f[^\s]*)\s+(%f[0-9]{1,2}),(%f[0-9]{1,2}),(%f[0-9]{1,2})/
597 $code =~ s
/\b(alignaddr)\s+(%[goli][0-7]),(%[goli][0-7]),(%[goli][0-7])/
598 &unalignaddr
($1,$2,$3,$4)