arch/arm64/include/asm/xor.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * arch/arm64/include/asm/xor.h
   4  *
   5  * Authors: Jackie Liu <liuyun01@kylinos.cn>
   6  * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
   7  */
   8
   9 #include <linux/hardirq.h>
  10 #include <asm-generic/xor.h>
  11 #include <asm/hwcap.h>
  12 #include <asm/neon.h>
  13
  14 #ifdef CONFIG_KERNEL_MODE_NEON
  15
  16 extern struct xor_block_template const xor_block_inner_neon;
  17
  18 static void
  19 xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
  20            const unsigned long * __restrict p2)
  21 {
  22         kernel_neon_begin();
  23         xor_block_inner_neon.do_2(bytes, p1, p2);
  24         kernel_neon_end();
  25 }
  26
  27 static void
  28 xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
  29            const unsigned long * __restrict p2,
  30            const unsigned long * __restrict p3)
  31 {
  32         kernel_neon_begin();
  33         xor_block_inner_neon.do_3(bytes, p1, p2, p3);
  34         kernel_neon_end();
  35 }
  36
  37 static void
  38 xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
  39            const unsigned long * __restrict p2,
  40            const unsigned long * __restrict p3,
  41            const unsigned long * __restrict p4)
  42 {
  43         kernel_neon_begin();
  44         xor_block_inner_neon.do_4(bytes, p1, p2, p3, p4);
  45         kernel_neon_end();
  46 }
  47
  48 static void
  49 xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
  50            const unsigned long * __restrict p2,
  51            const unsigned long * __restrict p3,
  52            const unsigned long * __restrict p4,
  53            const unsigned long * __restrict p5)
  54 {
  55         kernel_neon_begin();
  56         xor_block_inner_neon.do_5(bytes, p1, p2, p3, p4, p5);
  57         kernel_neon_end();
  58 }
  59
  60 static struct xor_block_template xor_block_arm64 = {
  61         .name   = "arm64_neon",
  62         .do_2   = xor_neon_2,
  63         .do_3   = xor_neon_3,
  64         .do_4   = xor_neon_4,
  65         .do_5   = xor_neon_5
  66 };
  67 #undef XOR_TRY_TEMPLATES
  68 #define XOR_TRY_TEMPLATES           \
  69         do {        \
  70                 xor_speed(&xor_block_8regs);    \
  71                 xor_speed(&xor_block_32regs);    \
  72                 if (cpu_has_neon()) { \
  73                         xor_speed(&xor_block_arm64);\
  74                 } \
  75         } while (0)
  76
  77 #endif /* ! CONFIG_KERNEL_MODE_NEON */