lib/raid6/mmx.c

   1 /* -*- linux-c -*- ------------------------------------------------------- *
   2  *
   3  *   Copyright 2002 H. Peter Anvin - All Rights Reserved
   4  *
   5  *   This program is free software; you can redistribute it and/or modify
   6  *   it under the terms of the GNU General Public License as published by
   7  *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
   8  *   Boston MA 02111-1307, USA; either version 2 of the License, or
   9  *   (at your option) any later version; incorporated herein by reference.
  10  *
  11  * ----------------------------------------------------------------------- */
  12
  13 /*
  14  * raid6/mmx.c
  15  *
  16  * MMX implementation of RAID-6 syndrome functions
  17  */
  18
  19 #if defined(__i386__) && !defined(__arch_um__)
  20
  21 #include <linux/raid/pq.h>
  22 #include "x86.h"
  23
  24 /* Shared with raid6/sse1.c */
  25 const struct raid6_mmx_constants {
  26         u64 x1d;
  27 } raid6_mmx_constants = {
  28         0x1d1d1d1d1d1d1d1dULL,
  29 };
  30
  31 static int raid6_have_mmx(void)
  32 {
  33         /* Not really "boot_cpu" but "all_cpus" */
  34         return boot_cpu_has(X86_FEATURE_MMX);
  35 }
  36
  37 /*
  38  * Plain MMX implementation
  39  */
  40 static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs)
  41 {
  42         u8 **dptr = (u8 **)ptrs;
  43         u8 *p, *q;
  44         int d, z, z0;
  45
  46         z0 = disks - 3;         /* Highest data disk */
  47         p = dptr[z0+1];         /* XOR parity */
  48         q = dptr[z0+2];         /* RS syndrome */
  49
  50         kernel_fpu_begin();
  51
  52         asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
  53         asm volatile("pxor %mm5,%mm5"); /* Zero temp */
  54
  55         for ( d = 0 ; d < bytes ; d += 8 ) {
  56                 asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
  57                 asm volatile("movq %mm2,%mm4"); /* Q[0] */
  58                 for ( z = z0-1 ; z >= 0 ; z-- ) {
  59                         asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d]));
  60                         asm volatile("pcmpgtb %mm4,%mm5");
  61                         asm volatile("paddb %mm4,%mm4");
  62                         asm volatile("pand %mm0,%mm5");
  63                         asm volatile("pxor %mm5,%mm4");
  64                         asm volatile("pxor %mm5,%mm5");
  65                         asm volatile("pxor %mm6,%mm2");
  66                         asm volatile("pxor %mm6,%mm4");
  67                 }
  68                 asm volatile("movq %%mm2,%0" : "=m" (p[d]));
  69                 asm volatile("pxor %mm2,%mm2");
  70                 asm volatile("movq %%mm4,%0" : "=m" (q[d]));
  71                 asm volatile("pxor %mm4,%mm4");
  72         }
  73
  74         kernel_fpu_end();
  75 }
  76
  77 const struct raid6_calls raid6_mmxx1 = {
  78         raid6_mmx1_gen_syndrome,
  79         raid6_have_mmx,
  80         "mmxx1",
  81         0
  82 };
  83
  84 /*
  85  * Unrolled-by-2 MMX implementation
  86  */
  87 static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs)
  88 {
  89         u8 **dptr = (u8 **)ptrs;
  90         u8 *p, *q;
  91         int d, z, z0;
  92
  93         z0 = disks - 3;         /* Highest data disk */
  94         p = dptr[z0+1];         /* XOR parity */
  95         q = dptr[z0+2];         /* RS syndrome */
  96
  97         kernel_fpu_begin();
  98
  99         asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
 100         asm volatile("pxor %mm5,%mm5"); /* Zero temp */
 101         asm volatile("pxor %mm7,%mm7"); /* Zero temp */
 102
 103         for ( d = 0 ; d < bytes ; d += 16 ) {
 104                 asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
 105                 asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8]));
 106                 asm volatile("movq %mm2,%mm4"); /* Q[0] */
 107                 asm volatile("movq %mm3,%mm6"); /* Q[1] */
 108                 for ( z = z0-1 ; z >= 0 ; z-- ) {
 109                         asm volatile("pcmpgtb %mm4,%mm5");
 110                         asm volatile("pcmpgtb %mm6,%mm7");
 111                         asm volatile("paddb %mm4,%mm4");
 112                         asm volatile("paddb %mm6,%mm6");
 113                         asm volatile("pand %mm0,%mm5");
 114                         asm volatile("pand %mm0,%mm7");
 115                         asm volatile("pxor %mm5,%mm4");
 116                         asm volatile("pxor %mm7,%mm6");
 117                         asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d]));
 118                         asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8]));
 119                         asm volatile("pxor %mm5,%mm2");
 120                         asm volatile("pxor %mm7,%mm3");
 121                         asm volatile("pxor %mm5,%mm4");
 122                         asm volatile("pxor %mm7,%mm6");
 123                         asm volatile("pxor %mm5,%mm5");
 124                         asm volatile("pxor %mm7,%mm7");
 125                 }
 126                 asm volatile("movq %%mm2,%0" : "=m" (p[d]));
 127                 asm volatile("movq %%mm3,%0" : "=m" (p[d+8]));
 128                 asm volatile("movq %%mm4,%0" : "=m" (q[d]));
 129                 asm volatile("movq %%mm6,%0" : "=m" (q[d+8]));
 130         }
 131
 132         kernel_fpu_end();
 133 }
 134
 135 const struct raid6_calls raid6_mmxx2 = {
 136         raid6_mmx2_gen_syndrome,
 137         raid6_have_mmx,
 138         "mmxx2",
 139         0
 140 };
 141
 142 #endif