arch/x86/entry/vdso/vgetrandom-chacha.S

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2022-2024 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
   4  */
   5
   6 #include <linux/linkage.h>
   7 #include <asm/frame.h>
   8
   9 .section        .rodata, "a"
  10 .align 16
  11 CONSTANTS:      .octa 0x6b20657479622d323320646e61707865
  12 .text
  13
  14 /*
  15  * Very basic SSE2 implementation of ChaCha20. Produces a given positive number
  16  * of blocks of output with a nonce of 0, taking an input key and 8-byte
  17  * counter. Importantly does not spill to the stack. Its arguments are:
  18  *
  19  *      rdi: output bytes
  20  *      rsi: 32-byte key input
  21  *      rdx: 8-byte counter input/output
  22  *      rcx: number of 64-byte blocks to write to output
  23  */
  24 SYM_FUNC_START(__arch_chacha20_blocks_nostack)
  25
  26 .set    output,         %rdi
  27 .set    key,            %rsi
  28 .set    counter,        %rdx
  29 .set    nblocks,        %rcx
  30 .set    i,              %al
  31 /* xmm registers are *not* callee-save. */
  32 .set    temp,           %xmm0
  33 .set    state0,         %xmm1
  34 .set    state1,         %xmm2
  35 .set    state2,         %xmm3
  36 .set    state3,         %xmm4
  37 .set    copy0,          %xmm5
  38 .set    copy1,          %xmm6
  39 .set    copy2,          %xmm7
  40 .set    copy3,          %xmm8
  41 .set    one,            %xmm9
  42
  43         /* copy0 = "expand 32-byte k" */
  44         movaps          CONSTANTS(%rip),copy0
  45         /* copy1,copy2 = key */
  46         movups          0x00(key),copy1
  47         movups          0x10(key),copy2
  48         /* copy3 = counter || zero nonce */
  49         movq            0x00(counter),copy3
  50         /* one = 1 || 0 */
  51         movq            $1,%rax
  52         movq            %rax,one
  53
  54 .Lblock:
  55         /* state0,state1,state2,state3 = copy0,copy1,copy2,copy3 */
  56         movdqa          copy0,state0
  57         movdqa          copy1,state1
  58         movdqa          copy2,state2
  59         movdqa          copy3,state3
  60
  61         movb            $10,i
  62 .Lpermute:
  63         /* state0 += state1, state3 = rotl32(state3 ^ state0, 16) */
  64         paddd           state1,state0
  65         pxor            state0,state3
  66         movdqa          state3,temp
  67         pslld           $16,temp
  68         psrld           $16,state3
  69         por             temp,state3
  70
  71         /* state2 += state3, state1 = rotl32(state1 ^ state2, 12) */
  72         paddd           state3,state2
  73         pxor            state2,state1
  74         movdqa          state1,temp
  75         pslld           $12,temp
  76         psrld           $20,state1
  77         por             temp,state1
  78
  79         /* state0 += state1, state3 = rotl32(state3 ^ state0, 8) */
  80         paddd           state1,state0
  81         pxor            state0,state3
  82         movdqa          state3,temp
  83         pslld           $8,temp
  84         psrld           $24,state3
  85         por             temp,state3
  86
  87         /* state2 += state3, state1 = rotl32(state1 ^ state2, 7) */
  88         paddd           state3,state2
  89         pxor            state2,state1
  90         movdqa          state1,temp
  91         pslld           $7,temp
  92         psrld           $25,state1
  93         por             temp,state1
  94
  95         /* state1[0,1,2,3] = state1[1,2,3,0] */
  96         pshufd          $0x39,state1,state1
  97         /* state2[0,1,2,3] = state2[2,3,0,1] */
  98         pshufd          $0x4e,state2,state2
  99         /* state3[0,1,2,3] = state3[3,0,1,2] */
 100         pshufd          $0x93,state3,state3
 101
 102         /* state0 += state1, state3 = rotl32(state3 ^ state0, 16) */
 103         paddd           state1,state0
 104         pxor            state0,state3
 105         movdqa          state3,temp
 106         pslld           $16,temp
 107         psrld           $16,state3
 108         por             temp,state3
 109
 110         /* state2 += state3, state1 = rotl32(state1 ^ state2, 12) */
 111         paddd           state3,state2
 112         pxor            state2,state1
 113         movdqa          state1,temp
 114         pslld           $12,temp
 115         psrld           $20,state1
 116         por             temp,state1
 117
 118         /* state0 += state1, state3 = rotl32(state3 ^ state0, 8) */
 119         paddd           state1,state0
 120         pxor            state0,state3
 121         movdqa          state3,temp
 122         pslld           $8,temp
 123         psrld           $24,state3
 124         por             temp,state3
 125
 126         /* state2 += state3, state1 = rotl32(state1 ^ state2, 7) */
 127         paddd           state3,state2
 128         pxor            state2,state1
 129         movdqa          state1,temp
 130         pslld           $7,temp
 131         psrld           $25,state1
 132         por             temp,state1
 133
 134         /* state1[0,1,2,3] = state1[3,0,1,2] */
 135         pshufd          $0x93,state1,state1
 136         /* state2[0,1,2,3] = state2[2,3,0,1] */
 137         pshufd          $0x4e,state2,state2
 138         /* state3[0,1,2,3] = state3[1,2,3,0] */
 139         pshufd          $0x39,state3,state3
 140
 141         decb            i
 142         jnz             .Lpermute
 143
 144         /* output0 = state0 + copy0 */
 145         paddd           copy0,state0
 146         movups          state0,0x00(output)
 147         /* output1 = state1 + copy1 */
 148         paddd           copy1,state1
 149         movups          state1,0x10(output)
 150         /* output2 = state2 + copy2 */
 151         paddd           copy2,state2
 152         movups          state2,0x20(output)
 153         /* output3 = state3 + copy3 */
 154         paddd           copy3,state3
 155         movups          state3,0x30(output)
 156
 157         /* ++copy3.counter */
 158         paddq           one,copy3
 159
 160         /* output += 64, --nblocks */
 161         addq            $64,output
 162         decq            nblocks
 163         jnz             .Lblock
 164
 165         /* counter = copy3.counter */
 166         movq            copy3,0x00(counter)
 167
 168         /* Zero out the potentially sensitive regs, in case nothing uses these again. */
 169         pxor            state0,state0
 170         pxor            state1,state1
 171         pxor            state2,state2
 172         pxor            state3,state3
 173         pxor            copy1,copy1
 174         pxor            copy2,copy2
 175         pxor            temp,temp
 176
 177         ret
 178 SYM_FUNC_END(__arch_chacha20_blocks_nostack)