arch/powerpc/crypto/ghashp10-ppc.pl

   1 #!/usr/bin/env perl
   2 # SPDX-License-Identifier: GPL-2.0
   3
   4 # This code is taken from the OpenSSL project but the author (Andy Polyakov)
   5 # has relicensed it under the GPLv2. Therefore this program is free software;
   6 # you can redistribute it and/or modify it under the terms of the GNU General
   7 # Public License version 2 as published by the Free Software Foundation.
   8 #
   9 # The original headers, including the original license headers, are
  10 # included below for completeness.
  11
  12 # ====================================================================
  13 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  14 # project. The module is, however, dual licensed under OpenSSL and
  15 # CRYPTOGAMS licenses depending on where you obtain it. For further
  16 # details see https://www.openssl.org/~appro/cryptogams/.
  17 # ====================================================================
  18 #
  19 # GHASH for PowerISA v2.07.
  20 #
  21 # July 2014
  22 #
  23 # Accurate performance measurements are problematic, because it's
  24 # always virtualized setup with possibly throttled processor.
  25 # Relative comparison is therefore more informative. This initial
  26 # version is ~2.1x slower than hardware-assisted AES-128-CTR, ~12x
  27 # faster than "4-bit" integer-only compiler-generated 64-bit code.
  28 # "Initial version" means that there is room for futher improvement.
  29
  30 $flavour=shift;
  31 $output =shift;
  32
  33 if ($flavour =~ /64/) {
  34         $SIZE_T=8;
  35         $LRSAVE=2*$SIZE_T;
  36         $STU="stdu";
  37         $POP="ld";
  38         $PUSH="std";
  39 } elsif ($flavour =~ /32/) {
  40         $SIZE_T=4;
  41         $LRSAVE=$SIZE_T;
  42         $STU="stwu";
  43         $POP="lwz";
  44         $PUSH="stw";
  45 } else { die "nonsense $flavour"; }
  46
  47 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  48 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
  49 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
  50 die "can't locate ppc-xlate.pl";
  51
  52 open STDOUT,"| $^X $xlate $flavour $output" || die "can't call $xlate: $!";
  53
  54 my ($Xip,$Htbl,$inp,$len)=map("r$_",(3..6));    # argument block
  55
  56 my ($Xl,$Xm,$Xh,$IN)=map("v$_",(0..3));
  57 my ($zero,$t0,$t1,$t2,$xC2,$H,$Hh,$Hl,$lemask)=map("v$_",(4..12));
  58 my ($Xl1,$Xm1,$Xh1,$IN1,$H2,$H2h,$H2l)=map("v$_",(13..19));
  59 my $vrsave="r12";
  60 my ($t4,$t5,$t6) = ($Hl,$H,$Hh);
  61
  62 $code=<<___;
  63 .machine        "any"
  64
  65 .text
  66
  67 .globl  .gcm_init_p10
  68         lis             r0,0xfff0
  69         li              r8,0x10
  70         mfspr           $vrsave,256
  71         li              r9,0x20
  72         mtspr           256,r0
  73         li              r10,0x30
  74         lvx_u           $H,0,r4                 # load H
  75         le?xor          r7,r7,r7
  76         le?addi         r7,r7,0x8               # need a vperm start with 08
  77         le?lvsr         5,0,r7
  78         le?vspltisb     6,0x0f
  79         le?vxor         5,5,6                   # set a b-endian mask
  80         le?vperm        $H,$H,$H,5
  81
  82         vspltisb        $xC2,-16                # 0xf0
  83         vspltisb        $t0,1                   # one
  84         vaddubm         $xC2,$xC2,$xC2          # 0xe0
  85         vxor            $zero,$zero,$zero
  86         vor             $xC2,$xC2,$t0           # 0xe1
  87         vsldoi          $xC2,$xC2,$zero,15      # 0xe1...
  88         vsldoi          $t1,$zero,$t0,1         # ...1
  89         vaddubm         $xC2,$xC2,$xC2          # 0xc2...
  90         vspltisb        $t2,7
  91         vor             $xC2,$xC2,$t1           # 0xc2....01
  92         vspltb          $t1,$H,0                # most significant byte
  93         vsl             $H,$H,$t0               # H<<=1
  94         vsrab           $t1,$t1,$t2             # broadcast carry bit
  95         vand            $t1,$t1,$xC2
  96         vxor            $H,$H,$t1               # twisted H
  97
  98         vsldoi          $H,$H,$H,8              # twist even more ...
  99         vsldoi          $xC2,$zero,$xC2,8       # 0xc2.0
 100         vsldoi          $Hl,$zero,$H,8          # ... and split
 101         vsldoi          $Hh,$H,$zero,8
 102
 103         stvx_u          $xC2,0,r3               # save pre-computed table
 104         stvx_u          $Hl,r8,r3
 105         stvx_u          $H, r9,r3
 106         stvx_u          $Hh,r10,r3
 107
 108         mtspr           256,$vrsave
 109         blr
 110         .long           0
 111         .byte           0,12,0x14,0,0,0,2,0
 112         .long           0
 113 .size   .gcm_init_p10,.-.gcm_init_p10
 114
 115 .globl  .gcm_init_htable
 116         lis             r0,0xfff0
 117         li              r8,0x10
 118         mfspr           $vrsave,256
 119         li              r9,0x20
 120         mtspr           256,r0
 121         li              r10,0x30
 122         lvx_u           $H,0,r4                 # load H
 123
 124         vspltisb        $xC2,-16                # 0xf0
 125         vspltisb        $t0,1                   # one
 126         vaddubm         $xC2,$xC2,$xC2          # 0xe0
 127         vxor            $zero,$zero,$zero
 128         vor             $xC2,$xC2,$t0           # 0xe1
 129         vsldoi          $xC2,$xC2,$zero,15      # 0xe1...
 130         vsldoi          $t1,$zero,$t0,1         # ...1
 131         vaddubm         $xC2,$xC2,$xC2          # 0xc2...
 132         vspltisb        $t2,7
 133         vor             $xC2,$xC2,$t1           # 0xc2....01
 134         vspltb          $t1,$H,0                # most significant byte
 135         vsl             $H,$H,$t0               # H<<=1
 136         vsrab           $t1,$t1,$t2             # broadcast carry bit
 137         vand            $t1,$t1,$xC2
 138         vxor            $IN,$H,$t1              # twisted H
 139
 140         vsldoi          $H,$IN,$IN,8            # twist even more ...
 141         vsldoi          $xC2,$zero,$xC2,8       # 0xc2.0
 142         vsldoi          $Hl,$zero,$H,8          # ... and split
 143         vsldoi          $Hh,$H,$zero,8
 144
 145         stvx_u          $xC2,0,r3               # save pre-computed table
 146         stvx_u          $Hl,r8,r3
 147         li              r8,0x40
 148         stvx_u          $H, r9,r3
 149         li              r9,0x50
 150         stvx_u          $Hh,r10,r3
 151         li              r10,0x60
 152
 153         vpmsumd         $Xl,$IN,$Hl             # H.lo·H.lo
 154         vpmsumd         $Xm,$IN,$H              # H.hi·H.lo+H.lo·H.hi
 155         vpmsumd         $Xh,$IN,$Hh             # H.hi·H.hi
 156
 157         vpmsumd         $t2,$Xl,$xC2            # 1st reduction phase
 158
 159         vsldoi          $t0,$Xm,$zero,8
 160         vsldoi          $t1,$zero,$Xm,8
 161         vxor            $Xl,$Xl,$t0
 162         vxor            $Xh,$Xh,$t1
 163
 164         vsldoi          $Xl,$Xl,$Xl,8
 165         vxor            $Xl,$Xl,$t2
 166
 167         vsldoi          $t1,$Xl,$Xl,8           # 2nd reduction phase
 168         vpmsumd         $Xl,$Xl,$xC2
 169         vxor            $t1,$t1,$Xh
 170         vxor            $IN1,$Xl,$t1
 171
 172         vsldoi          $H2,$IN1,$IN1,8
 173         vsldoi          $H2l,$zero,$H2,8
 174         vsldoi          $H2h,$H2,$zero,8
 175
 176         stvx_u          $H2l,r8,r3              # save H^2
 177         li              r8,0x70
 178         stvx_u          $H2,r9,r3
 179         li              r9,0x80
 180         stvx_u          $H2h,r10,r3
 181         li              r10,0x90
 182
 183         vpmsumd         $Xl,$IN,$H2l            # H.lo·H^2.lo
 184          vpmsumd        $Xl1,$IN1,$H2l          # H^2.lo·H^2.lo
 185         vpmsumd         $Xm,$IN,$H2             # H.hi·H^2.lo+H.lo·H^2.hi
 186          vpmsumd        $Xm1,$IN1,$H2           # H^2.hi·H^2.lo+H^2.lo·H^2.hi
 187         vpmsumd         $Xh,$IN,$H2h            # H.hi·H^2.hi
 188          vpmsumd        $Xh1,$IN1,$H2h          # H^2.hi·H^2.hi
 189
 190         vpmsumd         $t2,$Xl,$xC2            # 1st reduction phase
 191          vpmsumd        $t6,$Xl1,$xC2           # 1st reduction phase
 192
 193         vsldoi          $t0,$Xm,$zero,8
 194         vsldoi          $t1,$zero,$Xm,8
 195          vsldoi         $t4,$Xm1,$zero,8
 196          vsldoi         $t5,$zero,$Xm1,8
 197         vxor            $Xl,$Xl,$t0
 198         vxor            $Xh,$Xh,$t1
 199          vxor           $Xl1,$Xl1,$t4
 200          vxor           $Xh1,$Xh1,$t5
 201
 202         vsldoi          $Xl,$Xl,$Xl,8
 203          vsldoi         $Xl1,$Xl1,$Xl1,8
 204         vxor            $Xl,$Xl,$t2
 205          vxor           $Xl1,$Xl1,$t6
 206
 207         vsldoi          $t1,$Xl,$Xl,8           # 2nd reduction phase
 208          vsldoi         $t5,$Xl1,$Xl1,8         # 2nd reduction phase
 209         vpmsumd         $Xl,$Xl,$xC2
 210          vpmsumd        $Xl1,$Xl1,$xC2
 211         vxor            $t1,$t1,$Xh
 212          vxor           $t5,$t5,$Xh1
 213         vxor            $Xl,$Xl,$t1
 214          vxor           $Xl1,$Xl1,$t5
 215
 216         vsldoi          $H,$Xl,$Xl,8
 217          vsldoi         $H2,$Xl1,$Xl1,8
 218         vsldoi          $Hl,$zero,$H,8
 219         vsldoi          $Hh,$H,$zero,8
 220          vsldoi         $H2l,$zero,$H2,8
 221          vsldoi         $H2h,$H2,$zero,8
 222
 223         stvx_u          $Hl,r8,r3               # save H^3
 224         li              r8,0xa0
 225         stvx_u          $H,r9,r3
 226         li              r9,0xb0
 227         stvx_u          $Hh,r10,r3
 228         li              r10,0xc0
 229          stvx_u         $H2l,r8,r3              # save H^4
 230          stvx_u         $H2,r9,r3
 231          stvx_u         $H2h,r10,r3
 232
 233         mtspr           256,$vrsave
 234         blr
 235         .long           0
 236         .byte           0,12,0x14,0,0,0,2,0
 237         .long           0
 238 .size   .gcm_init_htable,.-.gcm_init_htable
 239
 240 .globl  .gcm_gmult_p10
 241         lis             r0,0xfff8
 242         li              r8,0x10
 243         mfspr           $vrsave,256
 244         li              r9,0x20
 245         mtspr           256,r0
 246         li              r10,0x30
 247         lvx_u           $IN,0,$Xip              # load Xi
 248
 249         lvx_u           $Hl,r8,$Htbl            # load pre-computed table
 250          le?lvsl        $lemask,r0,r0
 251         lvx_u           $H, r9,$Htbl
 252          le?vspltisb    $t0,0x07
 253         lvx_u           $Hh,r10,$Htbl
 254          le?vxor        $lemask,$lemask,$t0
 255         lvx_u           $xC2,0,$Htbl
 256          le?vperm       $IN,$IN,$IN,$lemask
 257         vxor            $zero,$zero,$zero
 258
 259         vpmsumd         $Xl,$IN,$Hl             # H.lo·Xi.lo
 260         vpmsumd         $Xm,$IN,$H              # H.hi·Xi.lo+H.lo·Xi.hi
 261         vpmsumd         $Xh,$IN,$Hh             # H.hi·Xi.hi
 262
 263         vpmsumd         $t2,$Xl,$xC2            # 1st phase
 264
 265         vsldoi          $t0,$Xm,$zero,8
 266         vsldoi          $t1,$zero,$Xm,8
 267         vxor            $Xl,$Xl,$t0
 268         vxor            $Xh,$Xh,$t1
 269
 270         vsldoi          $Xl,$Xl,$Xl,8
 271         vxor            $Xl,$Xl,$t2
 272
 273         vsldoi          $t1,$Xl,$Xl,8           # 2nd phase
 274         vpmsumd         $Xl,$Xl,$xC2
 275         vxor            $t1,$t1,$Xh
 276         vxor            $Xl,$Xl,$t1
 277
 278         le?vperm        $Xl,$Xl,$Xl,$lemask
 279         stvx_u          $Xl,0,$Xip              # write out Xi
 280
 281         mtspr           256,$vrsave
 282         blr
 283         .long           0
 284         .byte           0,12,0x14,0,0,0,2,0
 285         .long           0
 286 .size   .gcm_gmult_p10,.-.gcm_gmult_p10
 287
 288 .globl  .gcm_ghash_p10
 289         lis             r0,0xfff8
 290         li              r8,0x10
 291         mfspr           $vrsave,256
 292         li              r9,0x20
 293         mtspr           256,r0
 294         li              r10,0x30
 295         lvx_u           $Xl,0,$Xip              # load Xi
 296
 297         lvx_u           $Hl,r8,$Htbl            # load pre-computed table
 298          le?lvsl        $lemask,r0,r0
 299         lvx_u           $H, r9,$Htbl
 300          le?vspltisb    $t0,0x07
 301         lvx_u           $Hh,r10,$Htbl
 302          le?vxor        $lemask,$lemask,$t0
 303         lvx_u           $xC2,0,$Htbl
 304          le?vperm       $Xl,$Xl,$Xl,$lemask
 305         vxor            $zero,$zero,$zero
 306
 307         lvx_u           $IN,0,$inp
 308         addi            $inp,$inp,16
 309         subi            $len,$len,16
 310          le?vperm       $IN,$IN,$IN,$lemask
 311         vxor            $IN,$IN,$Xl
 312         b               Loop
 313
 314 .align  5
 315 Loop:
 316          subic          $len,$len,16
 317         vpmsumd         $Xl,$IN,$Hl             # H.lo·Xi.lo
 318          subfe.         r0,r0,r0                # borrow?-1:0
 319         vpmsumd         $Xm,$IN,$H              # H.hi·Xi.lo+H.lo·Xi.hi
 320          and            r0,r0,$len
 321         vpmsumd         $Xh,$IN,$Hh             # H.hi·Xi.hi
 322          add            $inp,$inp,r0
 323
 324         vpmsumd         $t2,$Xl,$xC2            # 1st phase
 325
 326         vsldoi          $t0,$Xm,$zero,8
 327         vsldoi          $t1,$zero,$Xm,8
 328         vxor            $Xl,$Xl,$t0
 329         vxor            $Xh,$Xh,$t1
 330
 331         vsldoi          $Xl,$Xl,$Xl,8
 332         vxor            $Xl,$Xl,$t2
 333          lvx_u          $IN,0,$inp
 334          addi           $inp,$inp,16
 335
 336         vsldoi          $t1,$Xl,$Xl,8           # 2nd phase
 337         vpmsumd         $Xl,$Xl,$xC2
 338          le?vperm       $IN,$IN,$IN,$lemask
 339         vxor            $t1,$t1,$Xh
 340         vxor            $IN,$IN,$t1
 341         vxor            $IN,$IN,$Xl
 342         beq             Loop                    # did $len-=16 borrow?
 343
 344         vxor            $Xl,$Xl,$t1
 345         le?vperm        $Xl,$Xl,$Xl,$lemask
 346         stvx_u          $Xl,0,$Xip              # write out Xi
 347
 348         mtspr           256,$vrsave
 349         blr
 350         .long           0
 351         .byte           0,12,0x14,0,0,0,4,0
 352         .long           0
 353 .size   .gcm_ghash_p10,.-.gcm_ghash_p10
 354
 355 .asciz  "GHASH for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
 356 .align  2
 357 ___
 358
 359 foreach (split("\n",$code)) {
 360         if ($flavour =~ /le$/o) {       # little-endian
 361             s/le\?//o           or
 362             s/be\?/#be#/o;
 363         } else {
 364             s/le\?/#le#/o       or
 365             s/be\?//o;
 366         }
 367         print $_,"\n";
 368 }
 369
 370 close STDOUT; # enforce flush