lib/raid6/mmx.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* -*- linux-c -*- ------------------------------------------------------- *
   3  *
   4  *   Copyright 2002 H. Peter Anvin - All Rights Reserved
   5  *
   6  * ----------------------------------------------------------------------- */
   7
   8 /*
   9  * raid6/mmx.c
  10  *
  11  * MMX implementation of RAID-6 syndrome functions
  12  */
  13
  14 #ifdef CONFIG_X86_32
  15
  16 #include <linux/raid/pq.h>
  17 #include "x86.h"
  18
  19 /* Shared with raid6/sse1.c */
  20 const struct raid6_mmx_constants {
  21         u64 x1d;
  22 } raid6_mmx_constants = {
  23         0x1d1d1d1d1d1d1d1dULL,
  24 };
  25
  26 static int raid6_have_mmx(void)
  27 {
  28         /* Not really "boot_cpu" but "all_cpus" */
  29         return boot_cpu_has(X86_FEATURE_MMX);
  30 }
  31
  32 /*
  33  * Plain MMX implementation
  34  */
  35 static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs)
  36 {
  37         u8 **dptr = (u8 **)ptrs;
  38         u8 *p, *q;
  39         int d, z, z0;
  40
  41         z0 = disks - 3;         /* Highest data disk */
  42         p = dptr[z0+1];         /* XOR parity */
  43         q = dptr[z0+2];         /* RS syndrome */
  44
  45         kernel_fpu_begin();
  46
  47         asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
  48         asm volatile("pxor %mm5,%mm5"); /* Zero temp */
  49
  50         for ( d = 0 ; d < bytes ; d += 8 ) {
  51                 asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
  52                 asm volatile("movq %mm2,%mm4"); /* Q[0] */
  53                 for ( z = z0-1 ; z >= 0 ; z-- ) {
  54                         asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d]));
  55                         asm volatile("pcmpgtb %mm4,%mm5");
  56                         asm volatile("paddb %mm4,%mm4");
  57                         asm volatile("pand %mm0,%mm5");
  58                         asm volatile("pxor %mm5,%mm4");
  59                         asm volatile("pxor %mm5,%mm5");
  60                         asm volatile("pxor %mm6,%mm2");
  61                         asm volatile("pxor %mm6,%mm4");
  62                 }
  63                 asm volatile("movq %%mm2,%0" : "=m" (p[d]));
  64                 asm volatile("pxor %mm2,%mm2");
  65                 asm volatile("movq %%mm4,%0" : "=m" (q[d]));
  66                 asm volatile("pxor %mm4,%mm4");
  67         }
  68
  69         kernel_fpu_end();
  70 }
  71
  72 const struct raid6_calls raid6_mmxx1 = {
  73         raid6_mmx1_gen_syndrome,
  74         NULL,                   /* XOR not yet implemented */
  75         raid6_have_mmx,
  76         "mmxx1",
  77         0
  78 };
  79
  80 /*
  81  * Unrolled-by-2 MMX implementation
  82  */
  83 static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs)
  84 {
  85         u8 **dptr = (u8 **)ptrs;
  86         u8 *p, *q;
  87         int d, z, z0;
  88
  89         z0 = disks - 3;         /* Highest data disk */
  90         p = dptr[z0+1];         /* XOR parity */
  91         q = dptr[z0+2];         /* RS syndrome */
  92
  93         kernel_fpu_begin();
  94
  95         asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
  96         asm volatile("pxor %mm5,%mm5"); /* Zero temp */
  97         asm volatile("pxor %mm7,%mm7"); /* Zero temp */
  98
  99         for ( d = 0 ; d < bytes ; d += 16 ) {
 100                 asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
 101                 asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8]));
 102                 asm volatile("movq %mm2,%mm4"); /* Q[0] */
 103                 asm volatile("movq %mm3,%mm6"); /* Q[1] */
 104                 for ( z = z0-1 ; z >= 0 ; z-- ) {
 105                         asm volatile("pcmpgtb %mm4,%mm5");
 106                         asm volatile("pcmpgtb %mm6,%mm7");
 107                         asm volatile("paddb %mm4,%mm4");
 108                         asm volatile("paddb %mm6,%mm6");
 109                         asm volatile("pand %mm0,%mm5");
 110                         asm volatile("pand %mm0,%mm7");
 111                         asm volatile("pxor %mm5,%mm4");
 112                         asm volatile("pxor %mm7,%mm6");
 113                         asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d]));
 114                         asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8]));
 115                         asm volatile("pxor %mm5,%mm2");
 116                         asm volatile("pxor %mm7,%mm3");
 117                         asm volatile("pxor %mm5,%mm4");
 118                         asm volatile("pxor %mm7,%mm6");
 119                         asm volatile("pxor %mm5,%mm5");
 120                         asm volatile("pxor %mm7,%mm7");
 121                 }
 122                 asm volatile("movq %%mm2,%0" : "=m" (p[d]));
 123                 asm volatile("movq %%mm3,%0" : "=m" (p[d+8]));
 124                 asm volatile("movq %%mm4,%0" : "=m" (q[d]));
 125                 asm volatile("movq %%mm6,%0" : "=m" (q[d+8]));
 126         }
 127
 128         kernel_fpu_end();
 129 }
 130
 131 const struct raid6_calls raid6_mmxx2 = {
 132         raid6_mmx2_gen_syndrome,
 133         NULL,                   /* XOR not yet implemented */
 134         raid6_have_mmx,
 135         "mmxx2",
 136         0
 137 };
 138
 139 #endif