usr/src/common/crypto/arcfour/amd64/arcfour-x86_64.pl

   1 #!/usr/bin/env perl
   2 #
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9 #
  10 # 2.22x RC4 tune-up:-) It should be noted though that my hand [as in
  11 # "hand-coded assembler"] doesn't stand for the whole improvement
  12 # coefficient. It turned out that eliminating RC4_CHAR from config
  13 # line results in ~40% improvement (yes, even for C implementation).
  14 # Presumably it has everything to do with AMD cache architecture and
  15 # RAW or whatever penalties. Once again! The module *requires* config
  16 # line *without* RC4_CHAR! As for coding "secret," I bet on partial
  17 # register arithmetics. For example instead of 'inc %r8; and $255,%r8'
  18 # I simply 'inc %r8b'. Even though optimization manual discourages
  19 # to operate on partial registers, it turned out to be the best bet.
  20 # At least for AMD... How IA32E would perform remains to be seen...
  21
  22 # As was shown by Marc Bevand reordering of couple of load operations
  23 # results in even higher performance gain of 3.3x:-) At least on
  24 # Opteron... For reference, 1x in this case is RC4_CHAR C-code
  25 # compiled with gcc 3.3.2, which performs at ~54MBps per 1GHz clock.
  26 # Latter means that if you want to *estimate* what to expect from
  27 # *your* Opteron, then multiply 54 by 3.3 and clock frequency in GHz.
  28
  29 # Intel P4 EM64T core was found to run the AMD64 code really slow...
  30 # The only way to achieve comparable performance on P4 was to keep
  31 # RC4_CHAR. Kind of ironic, huh? As it's apparently impossible to
  32 # compose blended code, which would perform even within 30% marginal
  33 # on either AMD and Intel platforms, I implement both cases. See
  34 # rc4_skey.c for further details...
  35
  36 # P4 EM64T core appears to be "allergic" to 64-bit inc/dec. Replacing
  37 # those with add/sub results in 50% performance improvement of folded
  38 # loop...
  39
  40 # As was shown by Zou Nanhai loop unrolling can improve Intel EM64T
  41 # performance by >30% [unlike P4 32-bit case that is]. But this is
  42 # provided that loads are reordered even more aggressively! Both code
  43 # pathes, AMD64 and EM64T, reorder loads in essentially same manner
  44 # as my IA-64 implementation. On Opteron this resulted in modest 5%
  45 # improvement [I had to test it], while final Intel P4 performance
  46 # achieves respectful 432MBps on 2.8GHz processor now. For reference.
  47 # If executed on Xeon, current RC4_CHAR code-path is 2.7x faster than
  48 # RC4_INT code-path. While if executed on Opteron, it's only 25%
  49 # slower than the RC4_INT one [meaning that if CPU µ-arch detection
  50 # is not implemented, then this final RC4_CHAR code-path should be
  51 # preferred, as it provides better *all-round* performance].
  52
  53 # Intel Core2 was observed to perform poorly on both code paths:-( It
  54 # apparently suffers from some kind of partial register stall, which
  55 # occurs in 64-bit mode only [as virtually identical 32-bit loop was
  56 # observed to outperform 64-bit one by almost 50%]. Adding two movzb to
  57 # cloop1 boosts its performance by 80%! This loop appears to be optimal
  58 # fit for Core2 and therefore the code was modified to skip cloop8 on
  59 # this CPU.
  60
  61 #
  62 # OpenSolaris OS modifications
  63 #
  64 # Sun elects to use this software under the BSD license.
  65 #
  66 # This source originates from OpenSSL file rc4-x86_64.pl at
  67 # ftp://ftp.openssl.org/snapshot/openssl-0.9.8-stable-SNAP-20080131.tar.gz
  68 # (presumably for future OpenSSL release 0.9.8h), with these changes:
  69 #
  70 # 1. Added some comments, "use strict", and declared all variables.
  71 #
  72 # 2. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
  73 # /usr/include/sys/asm_linkage.h.
  74 #
  75 # 3. Changed function name from RC4() to arcfour_crypt_asm() and RC4_set_key()
  76 # to arcfour_key_init(), and changed the parameter order for both to that
  77 # used by OpenSolaris.
  78 #
  79 # 4. The current method of using cpuid feature bits 20 (NX) or 28 (HTT) from
  80 # function OPENSSL_ia32_cpuid() to distinguish Intel/AMD does not work for
  81 # some newer AMD64 processors, as these bits are set on both Intel EM64T
  82 # processors and newer AMD64 processors.  I replaced this with C code
  83 # (function arcfour_crypt_on_intel()) to call cpuid_getvendor()
  84 # when executing in the kernel and getisax() when executing in userland.
  85 #
  86 # 5. Set a new field in the key structure, key->flag to 0 for AMD AMD64
  87 # and 1 for Intel EM64T.  This is to select the most-efficient arcfour_crypt()
  88 # function to use.
  89 #
  90 # 6. Removed x86_64-xlate.pl script (not needed for as(1) or gas(1) assemblers).
  91 #
  92 # 7. Removed unused RC4_CHAR, Lcloop1, and Lcloop8 code.
  93 #
  94 # 8. Added C function definitions for use by lint(1B).
  95 #
  96
  97 use strict;
  98 my ($code, $dat, $inp, $out, $len, $idx, $ido, $i, @XX, @TX, $YY, $TY);
  99 my $output = shift;
 100 open STDOUT,">$output";
 101
 102 #
 103 # Parameters
 104 #
 105
 106 # OpenSSL:
 107 # void RC4(RC4_KEY *key, unsigned long len, const unsigned char *indata,
 108 #       unsigned char *outdata);
 109 #$dat="%rdi";       # arg1
 110 #$len="%rsi";       # arg2
 111 #$inp="%rdx";       # arg3
 112 #$out="%rcx";       # arg4
 113
 114 # OpenSolaris:
 115 # void arcfour_crypt_asm(ARCFour_key *key, uchar_t *in, uchar_t *out,
 116 #       size_t len);
 117 $dat="%rdi";        # arg1
 118 $inp="%rsi";        # arg2
 119 $out="%rdx";        # arg3
 120 $len="%rcx";        # arg4
 121
 122 #
 123 # Register variables
 124 #
 125 # $XX[0] is key->i (aka key->x), $XX[1] is a temporary.
 126 # $TX[0] and $TX[1] are temporaries.
 127 # $YY is key->j (aka key->y).
 128 # $TY is a temporary.
 129 #
 130 @XX=("%r8","%r10");
 131 @TX=("%r9","%r11");
 132 $YY="%r12";
 133 $TY="%r13";
 134
 135 $code=<<___;
 136 #include <sys/asm_linkage.h>
 137
 138 ENTRY_NP(arcfour_crypt_asm)
 139         or      $len,$len # If (len == 0) return
 140         jne     .Lentry
 141         ret
 142 .Lentry:
 143         push    %r12
 144         push    %r13
 145
 146         / Set $dat to beginning of array, key->arr[0]
 147         add     \$8,$dat
 148         / Get key->j
 149         movl    -8($dat),$XX[0]#d
 150         / Get key->i
 151         movl    -4($dat),$YY#d
 152
 153         /
 154         / Use a 4-byte key schedule element array
 155         /
 156         inc     $XX[0]#b
 157         movl    ($dat,$XX[0],4),$TX[0]#d
 158         test    \$-8,$len
 159         jz      .Lloop1
 160         jmp     .Lloop8
 161
 162 .align  16
 163 .Lloop8:
 164 ___
 165 for ($i=0;$i<8;$i++) {
 166 $code.=<<___;
 167         add     $TX[0]#b,$YY#b
 168         mov     $XX[0],$XX[1]
 169         movl    ($dat,$YY,4),$TY#d
 170         ror     \$8,%rax                        # ror is redundant when $i=0
 171         inc     $XX[1]#b
 172         movl    ($dat,$XX[1],4),$TX[1]#d
 173         cmp     $XX[1],$YY
 174         movl    $TX[0]#d,($dat,$YY,4)
 175         cmove   $TX[0],$TX[1]
 176         movl    $TY#d,($dat,$XX[0],4)
 177         add     $TX[0]#b,$TY#b
 178         movb    ($dat,$TY,4),%al
 179 ___
 180 push(@TX,shift(@TX)); push(@XX,shift(@XX));     # "rotate" registers
 181 }
 182 $code.=<<___;
 183         ror     \$8,%rax
 184         sub     \$8,$len
 185
 186         xor     ($inp),%rax
 187         add     \$8,$inp
 188         mov     %rax,($out)
 189         add     \$8,$out
 190
 191         test    \$-8,$len
 192         jnz     .Lloop8
 193         cmp     \$0,$len
 194         jne     .Lloop1
 195
 196 .Lexit:
 197         /
 198         / Cleanup and exit code
 199         /
 200         / --i to undo ++i done at entry
 201         sub     \$1,$XX[0]#b
 202         / set key->i
 203         movl    $XX[0]#d,-8($dat)
 204         / set key->j
 205         movl    $YY#d,-4($dat)
 206
 207         pop     %r13
 208         pop     %r12
 209         ret
 210
 211 .align  16
 212 .Lloop1:
 213         add     $TX[0]#b,$YY#b
 214         movl    ($dat,$YY,4),$TY#d
 215         movl    $TX[0]#d,($dat,$YY,4)
 216         movl    $TY#d,($dat,$XX[0],4)
 217         add     $TY#b,$TX[0]#b
 218         inc     $XX[0]#b
 219         movl    ($dat,$TX[0],4),$TY#d
 220         movl    ($dat,$XX[0],4),$TX[0]#d
 221         xorb    ($inp),$TY#b
 222         inc     $inp
 223         movb    $TY#b,($out)
 224         inc     $out
 225         dec     $len
 226         jnz     .Lloop1
 227         jmp     .Lexit
 228
 229         ret
 230 SET_SIZE(arcfour_crypt_asm)
 231 ___
 232
 233
 234 #
 235 # Parameters
 236 #
 237
 238 # OpenSSL:
 239 # void RC4_set_key(RC4_KEY *key, int len, const unsigned char *data);
 240 #$dat="%rdi";       # arg1
 241 #$len="%rsi";       # arg2
 242 #$inp="%rdx";       # arg3
 243
 244 # OpenSolaris:
 245 # void arcfour_key_init(ARCFour_key *key, uchar_t *keyval, int keyvallen);
 246 $dat="%rdi";        # arg1
 247 $inp="%rsi";        # arg2
 248 $len="%rdx";        # arg3
 249
 250 # Temporaries
 251 $idx="%r8";
 252 $ido="%r9";
 253
 254 $code.=<<___;
 255         / int arcfour_crypt_on_intel(void);
 256 .extern arcfour_crypt_on_intel
 257
 258 ENTRY_NP(arcfour_key_init)
 259         / Find out if we're running on Intel or something else (e.g., AMD64).
 260         / This sets %eax to 1 for Intel, otherwise 0.
 261         push    %rdi            / Save arg1
 262         push    %rsi            / Save arg2
 263         push    %rdx            / Save arg3
 264         call    arcfour_crypt_on_intel
 265         pop     %rdx            / Restore arg3
 266         pop     %rsi            / Restore arg2
 267         pop     %rdi            / Restore arg1
 268         / Save return value in key->flag (1=Intel, 0=AMD)
 269         movl    %eax,1032($dat)
 270
 271         / Set $dat to beginning of array, key->arr[0]
 272         lea     8($dat),$dat
 273         lea     ($inp,$len),$inp
 274         neg     $len
 275         mov     $len,%rcx
 276
 277         xor     %eax,%eax
 278         xor     $ido,$ido
 279         xor     %r10,%r10
 280         xor     %r11,%r11
 281
 282         / Use a 4-byte data array
 283         jmp     .Lw1stloop
 284
 285 .align  16
 286 .Lw1stloop:
 287         / AMD64 (4-byte array)
 288         mov     %eax,($dat,%rax,4)
 289         add     \$1,%al
 290         jnc     .Lw1stloop
 291
 292         xor     $ido,$ido
 293         xor     $idx,$idx
 294
 295 .align  16
 296 .Lw2ndloop:
 297         mov     ($dat,$ido,4),%r10d
 298         add     ($inp,$len,1),$idx#b
 299         add     %r10b,$idx#b
 300         add     \$1,$len
 301         mov     ($dat,$idx,4),%r11d
 302         cmovz   %rcx,$len
 303         mov     %r10d,($dat,$idx,4)
 304         mov     %r11d,($dat,$ido,4)
 305         add     \$1,$ido#b
 306         jnc     .Lw2ndloop
 307
 308         / Exit code
 309         xor     %eax,%eax
 310         mov     %eax,-8($dat)
 311         mov     %eax,-4($dat)
 312
 313         ret
 314 SET_SIZE(arcfour_key_init)
 315 .asciz  "RC4 for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
 316 ___
 317
 318 $code =~ s/#([bwd])/$1/gm;
 319
 320 print $code;
 321
 322 close STDOUT;