module/icp/algs/blake3/blake3_generic.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Based on BLAKE3 v1.3.1, https://github.com/BLAKE3-team/BLAKE3
  24  * Copyright (c) 2019-2020 Samuel Neves and Jack O'Connor
  25  * Copyright (c) 2021-2022 Tino Reichardt <milky-zfs@mcmilk.de>
  26  */
  27
  28 #include <sys/simd.h>
  29 #include <sys/zfs_context.h>
  30 #include "blake3_impl.h"
  31
  32 #define rotr32(x, n)    (((x) >> (n)) | ((x) << (32 - (n))))
  33 static inline void g(uint32_t *state, size_t a, size_t b, size_t c, size_t d,
  34     uint32_t x, uint32_t y)
  35 {
  36         state[a] = state[a] + state[b] + x;
  37         state[d] = rotr32(state[d] ^ state[a], 16);
  38         state[c] = state[c] + state[d];
  39         state[b] = rotr32(state[b] ^ state[c], 12);
  40         state[a] = state[a] + state[b] + y;
  41         state[d] = rotr32(state[d] ^ state[a], 8);
  42         state[c] = state[c] + state[d];
  43         state[b] = rotr32(state[b] ^ state[c], 7);
  44 }
  45
  46 static inline void round_fn(uint32_t state[16], const uint32_t *msg,
  47     size_t round)
  48 {
  49         /* Select the message schedule based on the round. */
  50         const uint8_t *schedule = BLAKE3_MSG_SCHEDULE[round];
  51
  52         /* Mix the columns. */
  53         g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]);
  54         g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]);
  55         g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]);
  56         g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]);
  57
  58         /* Mix the rows. */
  59         g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]);
  60         g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]);
  61         g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]);
  62         g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]);
  63 }
  64
  65 static inline void compress_pre(uint32_t state[16], const uint32_t cv[8],
  66     const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
  67     uint64_t counter, uint8_t flags)
  68 {
  69         uint32_t block_words[16];
  70         block_words[0] = load32(block + 4 * 0);
  71         block_words[1] = load32(block + 4 * 1);
  72         block_words[2] = load32(block + 4 * 2);
  73         block_words[3] = load32(block + 4 * 3);
  74         block_words[4] = load32(block + 4 * 4);
  75         block_words[5] = load32(block + 4 * 5);
  76         block_words[6] = load32(block + 4 * 6);
  77         block_words[7] = load32(block + 4 * 7);
  78         block_words[8] = load32(block + 4 * 8);
  79         block_words[9] = load32(block + 4 * 9);
  80         block_words[10] = load32(block + 4 * 10);
  81         block_words[11] = load32(block + 4 * 11);
  82         block_words[12] = load32(block + 4 * 12);
  83         block_words[13] = load32(block + 4 * 13);
  84         block_words[14] = load32(block + 4 * 14);
  85         block_words[15] = load32(block + 4 * 15);
  86
  87         state[0] = cv[0];
  88         state[1] = cv[1];
  89         state[2] = cv[2];
  90         state[3] = cv[3];
  91         state[4] = cv[4];
  92         state[5] = cv[5];
  93         state[6] = cv[6];
  94         state[7] = cv[7];
  95         state[8] = BLAKE3_IV[0];
  96         state[9] = BLAKE3_IV[1];
  97         state[10] = BLAKE3_IV[2];
  98         state[11] = BLAKE3_IV[3];
  99         state[12] = counter_low(counter);
 100         state[13] = counter_high(counter);
 101         state[14] = (uint32_t)block_len;
 102         state[15] = (uint32_t)flags;
 103
 104         round_fn(state, &block_words[0], 0);
 105         round_fn(state, &block_words[0], 1);
 106         round_fn(state, &block_words[0], 2);
 107         round_fn(state, &block_words[0], 3);
 108         round_fn(state, &block_words[0], 4);
 109         round_fn(state, &block_words[0], 5);
 110         round_fn(state, &block_words[0], 6);
 111 }
 112
 113 static inline void blake3_compress_in_place_generic(uint32_t cv[8],
 114     const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
 115     uint64_t counter, uint8_t flags)
 116 {
 117         uint32_t state[16];
 118         compress_pre(state, cv, block, block_len, counter, flags);
 119         cv[0] = state[0] ^ state[8];
 120         cv[1] = state[1] ^ state[9];
 121         cv[2] = state[2] ^ state[10];
 122         cv[3] = state[3] ^ state[11];
 123         cv[4] = state[4] ^ state[12];
 124         cv[5] = state[5] ^ state[13];
 125         cv[6] = state[6] ^ state[14];
 126         cv[7] = state[7] ^ state[15];
 127 }
 128
 129 static inline void hash_one_generic(const uint8_t *input, size_t blocks,
 130     const uint32_t key[8], uint64_t counter, uint8_t flags,
 131     uint8_t flags_start, uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN])
 132 {
 133         uint32_t cv[8];
 134         memcpy(cv, key, BLAKE3_KEY_LEN);
 135         uint8_t block_flags = flags | flags_start;
 136         while (blocks > 0) {
 137                 if (blocks == 1) {
 138                         block_flags |= flags_end;
 139                 }
 140                 blake3_compress_in_place_generic(cv, input, BLAKE3_BLOCK_LEN,
 141                     counter, block_flags);
 142                 input = &input[BLAKE3_BLOCK_LEN];
 143                 blocks -= 1;
 144                 block_flags = flags;
 145         }
 146         store_cv_words(out, cv);
 147 }
 148
 149 static inline void blake3_compress_xof_generic(const uint32_t cv[8],
 150     const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
 151     uint64_t counter, uint8_t flags, uint8_t out[64])
 152 {
 153         uint32_t state[16];
 154         compress_pre(state, cv, block, block_len, counter, flags);
 155
 156         store32(&out[0 * 4], state[0] ^ state[8]);
 157         store32(&out[1 * 4], state[1] ^ state[9]);
 158         store32(&out[2 * 4], state[2] ^ state[10]);
 159         store32(&out[3 * 4], state[3] ^ state[11]);
 160         store32(&out[4 * 4], state[4] ^ state[12]);
 161         store32(&out[5 * 4], state[5] ^ state[13]);
 162         store32(&out[6 * 4], state[6] ^ state[14]);
 163         store32(&out[7 * 4], state[7] ^ state[15]);
 164         store32(&out[8 * 4], state[8] ^ cv[0]);
 165         store32(&out[9 * 4], state[9] ^ cv[1]);
 166         store32(&out[10 * 4], state[10] ^ cv[2]);
 167         store32(&out[11 * 4], state[11] ^ cv[3]);
 168         store32(&out[12 * 4], state[12] ^ cv[4]);
 169         store32(&out[13 * 4], state[13] ^ cv[5]);
 170         store32(&out[14 * 4], state[14] ^ cv[6]);
 171         store32(&out[15 * 4], state[15] ^ cv[7]);
 172 }
 173
 174 static inline void blake3_hash_many_generic(const uint8_t * const *inputs,
 175     size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter,
 176     boolean_t increment_counter, uint8_t flags, uint8_t flags_start,
 177     uint8_t flags_end, uint8_t *out)
 178 {
 179         while (num_inputs > 0) {
 180                 hash_one_generic(inputs[0], blocks, key, counter, flags,
 181                     flags_start, flags_end, out);
 182                 if (increment_counter) {
 183                         counter += 1;
 184                 }
 185                 inputs += 1;
 186                 num_inputs -= 1;
 187                 out = &out[BLAKE3_OUT_LEN];
 188         }
 189 }
 190
 191 /* the generic implementation is always okay */
 192 static boolean_t blake3_is_supported(void)
 193 {
 194         return (B_TRUE);
 195 }
 196
 197 const blake3_ops_t blake3_generic_impl = {
 198         .compress_in_place = blake3_compress_in_place_generic,
 199         .compress_xof = blake3_compress_xof_generic,
 200         .hash_many = blake3_hash_many_generic,
 201         .is_supported = blake3_is_supported,
 202         .degree = 4,
 203         .name = "generic"
 204 };