media/libtheora/lib/dec/x86/mmxfrag.c

   1 /********************************************************************
   2  *                                                                  *
   3  * THIS FILE IS PART OF THE OggTheora SOFTWARE CODEC SOURCE CODE.   *
   4  * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS     *
   5  * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
   6  * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING.       *
   7  *                                                                  *
   8  * THE Theora SOURCE CODE IS COPYRIGHT (C) 2002-2003                *
   9  * by the Xiph.Org Foundation http://www.xiph.org/                  *
  10  *                                                                  *
  11  ********************************************************************
  12
  13   function:
  14     last mod: $Id: mmxfrag.c 14345 2008-01-04 18:02:21Z tterribe $
  15
  16  ********************************************************************/
  17
  18 /*MMX acceleration of fragment reconstruction for motion compensation.
  19   Originally written by Rudolf Marek.
  20   Additional optimization by Nils Pipenbrinck.
  21   Note: Loops are unrolled for best performance.
  22   The iteration each instruction belongs to is marked in the comments as #i.*/
  23 #include "x86int.h"
  24
  25 #if defined(USE_ASM)
  26
  27 void oc_frag_recon_intra_mmx(unsigned char *_dst,int _dst_ystride,
  28  const ogg_int16_t *_residue){
  29   __asm__ __volatile__(
  30     /*Set mm0 to 0xFFFFFFFFFFFFFFFF.*/
  31     "pcmpeqw %%mm0,%%mm0\n\t"
  32     /*#0 Load low residue.*/
  33     "movq 0*8(%[residue]),%%mm1\n\t"
  34     /*#0 Load high residue.*/
  35     "movq 1*8(%[residue]),%%mm2\n\t"
  36     /*Set mm0 to 0x8000800080008000.*/
  37     "psllw $15,%%mm0\n\t"
  38     /*#1 Load low residue.*/
  39     "movq 2*8(%[residue]),%%mm3\n\t"
  40     /*#1 Load high residue.*/
  41     "movq 3*8(%[residue]),%%mm4\n\t"
  42     /*Set mm0 to 0x0080008000800080.*/
  43     "psrlw $8,%%mm0\n\t"
  44     /*#2 Load low residue.*/
  45     "movq 4*8(%[residue]),%%mm5\n\t"
  46     /*#2 Load high residue.*/
  47     "movq 5*8(%[residue]),%%mm6\n\t"
  48     /*#0 Bias low  residue.*/
  49     "paddsw %%mm0,%%mm1\n\t"
  50     /*#0 Bias high residue.*/
  51     "paddsw %%mm0,%%mm2\n\t"
  52     /*#0 Pack to byte.*/
  53     "packuswb %%mm2,%%mm1\n\t"
  54     /*#1 Bias low  residue.*/
  55     "paddsw %%mm0,%%mm3\n\t"
  56     /*#1 Bias high residue.*/
  57     "paddsw %%mm0,%%mm4\n\t"
  58     /*#1 Pack to byte.*/
  59     "packuswb %%mm4,%%mm3\n\t"
  60     /*#2 Bias low  residue.*/
  61     "paddsw %%mm0,%%mm5\n\t"
  62     /*#2 Bias high residue.*/
  63     "paddsw %%mm0,%%mm6\n\t"
  64     /*#2 Pack to byte.*/
  65     "packuswb %%mm6,%%mm5\n\t"
  66     /*#0 Write row.*/
  67     "movq %%mm1,(%[dst])\n\t"
  68     /*#1 Write row.*/
  69     "movq %%mm3,(%[dst],%[dst_ystride])\n\t"
  70     /*#2 Write row.*/
  71     "movq %%mm5,(%[dst],%[dst_ystride],2)\n\t"
  72     /*#3 Load low residue.*/
  73     "movq 6*8(%[residue]),%%mm1\n\t"
  74     /*#3 Load high residue.*/
  75     "movq 7*8(%[residue]),%%mm2\n\t"
  76     /*#4 Load high residue.*/
  77     "movq 8*8(%[residue]),%%mm3\n\t"
  78     /*#4 Load high residue.*/
  79     "movq 9*8(%[residue]),%%mm4\n\t"
  80     /*#5 Load high residue.*/
  81     "movq 10*8(%[residue]),%%mm5\n\t"
  82     /*#5 Load high residue.*/
  83     "movq 11*8(%[residue]),%%mm6\n\t"
  84     /*#3 Bias low  residue.*/
  85     "paddsw %%mm0,%%mm1\n\t"
  86     /*#3 Bias high residue.*/
  87     "paddsw %%mm0,%%mm2\n\t"
  88     /*#3 Pack to byte.*/
  89     "packuswb %%mm2,%%mm1\n\t"
  90     /*#4 Bias low  residue.*/
  91     "paddsw %%mm0,%%mm3\n\t"
  92     /*#4 Bias high residue.*/
  93     "paddsw %%mm0,%%mm4\n\t"
  94     /*#4 Pack to byte.*/
  95     "packuswb %%mm4,%%mm3\n\t"
  96     /*#5 Bias low  residue.*/
  97     "paddsw %%mm0,%%mm5\n\t"
  98     /*#5 Bias high residue.*/
  99     "paddsw %%mm0,%%mm6\n\t"
 100     /*#5 Pack to byte.*/
 101     "packuswb %%mm6,%%mm5\n\t"
 102     /*#3 Write row.*/
 103     "movq %%mm1,(%[dst],%[dst_ystride3])\n\t"
 104     /*#4 Write row.*/
 105     "movq %%mm3,(%[dst4])\n\t"
 106     /*#5 Write row.*/
 107     "movq %%mm5,(%[dst4],%[dst_ystride])\n\t"
 108     /*#6 Load low residue.*/
 109     "movq 12*8(%[residue]),%%mm1\n\t"
 110     /*#6 Load high residue.*/
 111     "movq 13*8(%[residue]),%%mm2\n\t"
 112     /*#7 Load low residue.*/
 113     "movq 14*8(%[residue]),%%mm3\n\t"
 114     /*#7 Load high residue.*/
 115     "movq 15*8(%[residue]),%%mm4\n\t"
 116     /*#6 Bias low  residue.*/
 117     "paddsw %%mm0,%%mm1\n\t"
 118     /*#6 Bias high residue.*/
 119     "paddsw %%mm0,%%mm2\n\t"
 120     /*#6 Pack to byte.*/
 121     "packuswb %%mm2,%%mm1\n\t"
 122     /*#7 Bias low  residue.*/
 123     "paddsw %%mm0,%%mm3\n\t"
 124     /*#7 Bias high residue.*/
 125     "paddsw %%mm0,%%mm4\n\t"
 126     /*#7 Pack to byte.*/
 127     "packuswb %%mm4,%%mm3\n\t"
 128     /*#6 Write row.*/
 129     "movq %%mm1,(%[dst4],%[dst_ystride],2)\n\t"
 130     /*#7 Write row.*/
 131     "movq %%mm3,(%[dst4],%[dst_ystride3])\n\t"
 132     :
 133     :[residue]"r"(_residue),
 134      [dst]"r"(_dst),
 135      [dst4]"r"(_dst+(_dst_ystride<<2)),
 136      [dst_ystride]"r"((long)_dst_ystride),
 137      [dst_ystride3]"r"((long)_dst_ystride*3)
 138     :"memory"
 139   );
 140 }
 141
 142 void oc_frag_recon_inter_mmx(unsigned char *_dst,int _dst_ystride,
 143  const unsigned char *_src,int _src_ystride,const ogg_int16_t *_residue){
 144   int i;
 145   /*Zero mm0.*/
 146   __asm__ __volatile__("pxor %%mm0,%%mm0\n\t"::);
 147   for(i=4;i-->0;){
 148     __asm__ __volatile__(
 149       /*#0 Load source.*/
 150       "movq (%[src]),%%mm3\n\t"
 151       /*#1 Load source.*/
 152       "movq (%[src],%[src_ystride]),%%mm7\n\t"
 153       /*#0 Get copy of src.*/
 154       "movq %%mm3,%%mm4\n\t"
 155       /*#0 Expand high source.*/
 156       "punpckhbw %%mm0,%%mm4\n\t"
 157       /*#0 Expand low  source.*/
 158       "punpcklbw %%mm0,%%mm3\n\t"
 159       /*#0 Add residue high.*/
 160       "paddsw 8(%[residue]),%%mm4\n\t"
 161       /*#1 Get copy of src.*/
 162       "movq %%mm7,%%mm2\n\t"
 163       /*#0 Add residue low.*/
 164       "paddsw (%[residue]), %%mm3\n\t"
 165       /*#1 Expand high source.*/
 166       "punpckhbw %%mm0,%%mm2\n\t"
 167       /*#0 Pack final row pixels.*/
 168       "packuswb %%mm4,%%mm3\n\t"
 169       /*#1 Expand low  source.*/
 170       "punpcklbw %%mm0,%%mm7\n\t"
 171       /*#1 Add residue low.*/
 172       "paddsw 16(%[residue]),%%mm7\n\t"
 173       /*#1 Add residue high.*/
 174       "paddsw 24(%[residue]),%%mm2\n\t"
 175       /*Advance residue.*/
 176       "lea 32(%[residue]),%[residue]\n\t"
 177       /*#1 Pack final row pixels.*/
 178       "packuswb %%mm2,%%mm7\n\t"
 179       /*Advance src.*/
 180       "lea (%[src],%[src_ystride],2),%[src]\n\t"
 181       /*#0 Write row.*/
 182       "movq %%mm3,(%[dst])\n\t"
 183       /*#1 Write row.*/
 184       "movq %%mm7,(%[dst],%[dst_ystride])\n\t"
 185       /*Advance dst.*/
 186       "lea (%[dst],%[dst_ystride],2),%[dst]\n\t"
 187       :[residue]"+r"(_residue),[dst]"+r"(_dst),[src]"+r"(_src)
 188       :[dst_ystride]"r"((long)_dst_ystride),
 189        [src_ystride]"r"((long)_src_ystride)
 190       :"memory"
 191     );
 192   }
 193 }
 194
 195 void oc_frag_recon_inter2_mmx(unsigned char *_dst,int _dst_ystride,
 196  const unsigned char *_src1,int _src1_ystride,const unsigned char *_src2,
 197  int _src2_ystride,const ogg_int16_t *_residue){
 198   int i;
 199   /*NOTE: This assumes that
 200      _dst_ystride==_src1_ystride&&_dst_ystride==_src2_ystride.
 201     This is currently always the case, but a slower fallback version will need
 202      to be written if it ever is not.*/
 203   /*Zero mm7.*/
 204   __asm__ __volatile__("pxor %%mm7,%%mm7\n\t"::);
 205   for(i=4;i-->0;){
 206     __asm__ __volatile__(
 207       /*#0 Load src1.*/
 208       "movq (%[src1]),%%mm0\n\t"
 209       /*#0 Load src2.*/
 210       "movq (%[src2]),%%mm2\n\t"
 211       /*#0 Copy src1.*/
 212       "movq %%mm0,%%mm1\n\t"
 213       /*#0 Copy src2.*/
 214       "movq %%mm2,%%mm3\n\t"
 215       /*#1 Load src1.*/
 216       "movq (%[src1],%[ystride]),%%mm4\n\t"
 217       /*#0 Unpack lower src1.*/
 218       "punpcklbw %%mm7,%%mm0\n\t"
 219       /*#1 Load src2.*/
 220       "movq (%[src2],%[ystride]),%%mm5\n\t"
 221       /*#0 Unpack higher src1.*/
 222       "punpckhbw %%mm7,%%mm1\n\t"
 223       /*#0 Unpack lower src2.*/
 224       "punpcklbw %%mm7,%%mm2\n\t"
 225       /*#0 Unpack higher src2.*/
 226       "punpckhbw %%mm7,%%mm3\n\t"
 227       /*Advance src1 ptr.*/
 228       "lea (%[src1],%[ystride],2),%[src1]\n\t"
 229       /*Advance src2 ptr.*/
 230       "lea (%[src2],%[ystride],2),%[src2]\n\t"
 231       /*#0 Lower src1+src2.*/
 232       "paddsw %%mm2,%%mm0\n\t"
 233       /*#0 Higher src1+src2.*/
 234       "paddsw %%mm3,%%mm1\n\t"
 235       /*#1 Copy src1.*/
 236       "movq %%mm4,%%mm2\n\t"
 237       /*#0 Build lo average.*/
 238       "psraw $1,%%mm0\n\t"
 239       /*#1 Copy src2.*/
 240       "movq %%mm5,%%mm3\n\t"
 241       /*#1 Unpack lower src1.*/
 242       "punpcklbw %%mm7,%%mm4\n\t"
 243       /*#0 Build hi average.*/
 244       "psraw $1,%%mm1\n\t"
 245       /*#1 Unpack higher src1.*/
 246       "punpckhbw %%mm7,%%mm2\n\t"
 247       /*#0 low+=residue.*/
 248       "paddsw (%[residue]),%%mm0\n\t"
 249       /*#1 Unpack lower src2.*/
 250       "punpcklbw %%mm7,%%mm5\n\t"
 251       /*#0 high+=residue.*/
 252       "paddsw 8(%[residue]),%%mm1\n\t"
 253       /*#1 Unpack higher src2.*/
 254       "punpckhbw %%mm7,%%mm3\n\t"
 255       /*#1 Lower src1+src2.*/
 256       "paddsw %%mm4,%%mm5\n\t"
 257       /*#0 Pack and saturate.*/
 258       "packuswb %%mm1,%%mm0\n\t"
 259       /*#1 Higher src1+src2.*/
 260       "paddsw %%mm2,%%mm3\n\t"
 261       /*#0 Write row.*/
 262       "movq %%mm0,(%[dst])\n\t"
 263       /*#1 Build lo average.*/
 264       "psraw $1,%%mm5\n\t"
 265       /*#1 Build hi average.*/
 266       "psraw $1,%%mm3\n\t"
 267       /*#1 low+=residue.*/
 268       "paddsw 16(%[residue]),%%mm5\n\t"
 269       /*#1 high+=residue.*/
 270       "paddsw 24(%[residue]),%%mm3\n\t"
 271       /*#1 Pack and saturate.*/
 272       "packuswb  %%mm3,%%mm5\n\t"
 273       /*#1 Write row ptr.*/
 274       "movq %%mm5,(%[dst],%[ystride])\n\t"
 275       /*Advance residue ptr.*/
 276       "add $32,%[residue]\n\t"
 277       /*Advance dest ptr.*/
 278       "lea (%[dst],%[ystride],2),%[dst]\n\t"
 279      :[dst]"+r"(_dst),[residue]"+r"(_residue),
 280       [src1]"+r"(_src1),[src2]"+r"(_src2)
 281      :[ystride]"r"((long)_dst_ystride)
 282      :"memory"
 283     );
 284   }
 285 }
 286
 287 void oc_restore_fpu_mmx(void){
 288   __asm__ __volatile__("emms\n\t");
 289 }
 290 #endif