drivers/md/raid6mmx.c

   1 /* -*- linux-c -*- ------------------------------------------------------- *
   2  *
   3  *   Copyright 2002 H. Peter Anvin - All Rights Reserved
   4  *
   5  *   This program is free software; you can redistribute it and/or modify
   6  *   it under the terms of the GNU General Public License as published by
   7  *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
   8  *   Bostom MA 02111-1307, USA; either version 2 of the License, or
   9  *   (at your option) any later version; incorporated herein by reference.
  10  *
  11  * ----------------------------------------------------------------------- */
  12
  13 /*
  14  * raid6mmx.c
  15  *
  16  * MMX implementation of RAID-6 syndrome functions
  17  */
  18
  19 #if defined(__i386__)
  20
  21 #include "raid6.h"
  22 #include "raid6x86.h"
  23
  24 /* Shared with raid6sse1.c */
  25 const struct raid6_mmx_constants {
  26         u64 x1d;
  27 } raid6_mmx_constants = {
  28         0x1d1d1d1d1d1d1d1dULL,
  29 };
  30
  31 static int raid6_have_mmx(void)
  32 {
  33 #ifdef __KERNEL__
  34         /* Not really "boot_cpu" but "all_cpus" */
  35         return boot_cpu_has(X86_FEATURE_MMX);
  36 #else
  37         /* User space test code */
  38         u32 features = cpuid_features();
  39         return ( (features & (1<<23)) == (1<<23) );
  40 #endif
  41 }
  42
  43 /*
  44  * Plain MMX implementation
  45  */
  46 static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs)
  47 {
  48         u8 **dptr = (u8 **)ptrs;
  49         u8 *p, *q;
  50         int d, z, z0;
  51         raid6_mmx_save_t sa;
  52
  53         z0 = disks - 3;         /* Highest data disk */
  54         p = dptr[z0+1];         /* XOR parity */
  55         q = dptr[z0+2];         /* RS syndrome */
  56
  57         raid6_before_mmx(&sa);
  58
  59         asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
  60         asm volatile("pxor %mm5,%mm5"); /* Zero temp */
  61
  62         for ( d = 0 ; d < bytes ; d += 8 ) {
  63                 asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
  64                 asm volatile("movq %mm2,%mm4"); /* Q[0] */
  65                 for ( z = z0-1 ; z >= 0 ; z-- ) {
  66                         asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d]));
  67                         asm volatile("pcmpgtb %mm4,%mm5");
  68                         asm volatile("paddb %mm4,%mm4");
  69                         asm volatile("pand %mm0,%mm5");
  70                         asm volatile("pxor %mm5,%mm4");
  71                         asm volatile("pxor %mm5,%mm5");
  72                         asm volatile("pxor %mm6,%mm2");
  73                         asm volatile("pxor %mm6,%mm4");
  74                 }
  75                 asm volatile("movq %%mm2,%0" : "=m" (p[d]));
  76                 asm volatile("pxor %mm2,%mm2");
  77                 asm volatile("movq %%mm4,%0" : "=m" (q[d]));
  78                 asm volatile("pxor %mm4,%mm4");
  79         }
  80
  81         raid6_after_mmx(&sa);
  82 }
  83
  84 const struct raid6_calls raid6_mmxx1 = {
  85         raid6_mmx1_gen_syndrome,
  86         raid6_have_mmx,
  87         "mmxx1",
  88         0
  89 };
  90
  91 /*
  92  * Unrolled-by-2 MMX implementation
  93  */
  94 static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs)
  95 {
  96         u8 **dptr = (u8 **)ptrs;
  97         u8 *p, *q;
  98         int d, z, z0;
  99         raid6_mmx_save_t sa;
 100
 101         z0 = disks - 3;         /* Highest data disk */
 102         p = dptr[z0+1];         /* XOR parity */
 103         q = dptr[z0+2];         /* RS syndrome */
 104
 105         raid6_before_mmx(&sa);
 106
 107         asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
 108         asm volatile("pxor %mm5,%mm5"); /* Zero temp */
 109         asm volatile("pxor %mm7,%mm7"); /* Zero temp */
 110
 111         for ( d = 0 ; d < bytes ; d += 16 ) {
 112                 asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
 113                 asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8]));
 114                 asm volatile("movq %mm2,%mm4"); /* Q[0] */
 115                 asm volatile("movq %mm3,%mm6"); /* Q[1] */
 116                 for ( z = z0-1 ; z >= 0 ; z-- ) {
 117                         asm volatile("pcmpgtb %mm4,%mm5");
 118                         asm volatile("pcmpgtb %mm6,%mm7");
 119                         asm volatile("paddb %mm4,%mm4");
 120                         asm volatile("paddb %mm6,%mm6");
 121                         asm volatile("pand %mm0,%mm5");
 122                         asm volatile("pand %mm0,%mm7");
 123                         asm volatile("pxor %mm5,%mm4");
 124                         asm volatile("pxor %mm7,%mm6");
 125                         asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d]));
 126                         asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8]));
 127                         asm volatile("pxor %mm5,%mm2");
 128                         asm volatile("pxor %mm7,%mm3");
 129                         asm volatile("pxor %mm5,%mm4");
 130                         asm volatile("pxor %mm7,%mm6");
 131                         asm volatile("pxor %mm5,%mm5");
 132                         asm volatile("pxor %mm7,%mm7");
 133                 }
 134                 asm volatile("movq %%mm2,%0" : "=m" (p[d]));
 135                 asm volatile("movq %%mm3,%0" : "=m" (p[d+8]));
 136                 asm volatile("movq %%mm4,%0" : "=m" (q[d]));
 137                 asm volatile("movq %%mm6,%0" : "=m" (q[d+8]));
 138         }
 139
 140         raid6_after_mmx(&sa);
 141 }
 142
 143 const struct raid6_calls raid6_mmxx2 = {
 144         raid6_mmx2_gen_syndrome,
 145         raid6_have_mmx,
 146         "mmxx2",
 147         0
 148 };
 149
 150 #endif